mirror of
https://github.com/SigNoz/signoz.git
synced 2026-07-29 17:20:40 +01:00
Compare commits
33 Commits
v2-transpi
...
issue_5601
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d6d09549f0 | ||
|
|
d71c0c938a | ||
|
|
e911223b1d | ||
|
|
4e45620b72 | ||
|
|
a9506f1354 | ||
|
|
cb3cc3de56 | ||
|
|
017e62814a | ||
|
|
e2e7caf1ca | ||
|
|
f10f526249 | ||
|
|
a5350682c3 | ||
|
|
a7b74005ea | ||
|
|
787e1be974 | ||
|
|
19ee51be66 | ||
|
|
b2c5af428e | ||
|
|
093f9b41c4 | ||
|
|
596e128005 | ||
|
|
2e1da92367 | ||
|
|
1e8d72e8fa | ||
|
|
cce080e1ae | ||
|
|
deca060a4d | ||
|
|
03c7e524e7 | ||
|
|
815dc7d88b | ||
|
|
f50d9199fe | ||
|
|
31efe177a4 | ||
|
|
d502d12ac3 | ||
|
|
bd9f15a716 | ||
|
|
813ef988c9 | ||
|
|
40e6799285 | ||
|
|
1caa60a3cd | ||
|
|
3f781f0083 | ||
|
|
6aec05cf7a | ||
|
|
683a52f35a | ||
|
|
e924fa1e62 |
1
.github/workflows/integrationci.yaml
vendored
1
.github/workflows/integrationci.yaml
vendored
@@ -53,6 +53,7 @@ jobs:
|
||||
- queriermetrics
|
||||
- querierscalar
|
||||
- queriercommon
|
||||
- querierai
|
||||
- rawexportdata
|
||||
- promqlconformance
|
||||
- querierauthz
|
||||
|
||||
@@ -2,9 +2,9 @@ package main
|
||||
|
||||
import (
|
||||
"github.com/SigNoz/signoz/pkg/metercollector"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
)
|
||||
@@ -25,8 +25,8 @@ var meterConfigs = []metercollector.Config{
|
||||
Name: zeustypes.MeterLogSize,
|
||||
Unit: zeustypes.MeterUnitBytes,
|
||||
Aggregation: zeustypes.MeterAggregationSum,
|
||||
DBName: telemetrylogs.DBName,
|
||||
TableName: telemetrylogs.LogsV2LocalTableName,
|
||||
DBName: logstelemetryschema.DBName,
|
||||
TableName: logstelemetryschema.LogsV2LocalTableName,
|
||||
DefaultRetentionDays: retentiontypes.DefaultLogsRetentionDays,
|
||||
},
|
||||
},
|
||||
@@ -36,8 +36,8 @@ var meterConfigs = []metercollector.Config{
|
||||
Name: zeustypes.MeterSpanSize,
|
||||
Unit: zeustypes.MeterUnitBytes,
|
||||
Aggregation: zeustypes.MeterAggregationSum,
|
||||
DBName: telemetrytraces.DBName,
|
||||
TableName: telemetrytraces.SpanIndexV3LocalTableName,
|
||||
DBName: tracestelemetryschema.DBName,
|
||||
TableName: tracestelemetryschema.SpanIndexV3LocalTableName,
|
||||
DefaultRetentionDays: retentiontypes.DefaultTracesRetentionDays,
|
||||
},
|
||||
},
|
||||
@@ -47,8 +47,8 @@ var meterConfigs = []metercollector.Config{
|
||||
Name: zeustypes.MeterDatapointCount,
|
||||
Unit: zeustypes.MeterUnitCount,
|
||||
Aggregation: zeustypes.MeterAggregationSum,
|
||||
DBName: telemetrymetrics.DBName,
|
||||
TableName: telemetrymetrics.SamplesV4LocalTableName,
|
||||
DBName: metricstelemetryschema.DBName,
|
||||
TableName: metricstelemetryschema.SamplesV4LocalTableName,
|
||||
DefaultRetentionDays: retentiontypes.DefaultMetricsRetentionDays,
|
||||
},
|
||||
},
|
||||
|
||||
@@ -6899,6 +6899,7 @@ components:
|
||||
Querybuildertypesv5QueryEnvelope:
|
||||
discriminator:
|
||||
mapping:
|
||||
builder_ai_query: '#/components/schemas/Querybuildertypesv5QueryEnvelopeBuilderAI'
|
||||
builder_formula: '#/components/schemas/Querybuildertypesv5QueryEnvelopeFormula'
|
||||
builder_query: '#/components/schemas/Querybuildertypesv5QueryEnvelopeBuilder'
|
||||
builder_trace_operator: '#/components/schemas/Querybuildertypesv5QueryEnvelopeTraceOperator'
|
||||
@@ -6907,6 +6908,7 @@ components:
|
||||
propertyName: type
|
||||
oneOf:
|
||||
- $ref: '#/components/schemas/Querybuildertypesv5QueryEnvelopeBuilder'
|
||||
- $ref: '#/components/schemas/Querybuildertypesv5QueryEnvelopeBuilderAI'
|
||||
- $ref: '#/components/schemas/Querybuildertypesv5QueryEnvelopeFormula'
|
||||
- $ref: '#/components/schemas/Querybuildertypesv5QueryEnvelopeTraceOperator'
|
||||
- $ref: '#/components/schemas/Querybuildertypesv5QueryEnvelopePromQL'
|
||||
@@ -6921,6 +6923,15 @@ components:
|
||||
required:
|
||||
- type
|
||||
type: object
|
||||
Querybuildertypesv5QueryEnvelopeBuilderAI:
|
||||
properties:
|
||||
spec:
|
||||
$ref: '#/components/schemas/Querybuildertypesv5QueryBuilderQueryGithubComSigNozSignozPkgTypesQuerybuildertypesQuerybuildertypesv5TraceAggregation'
|
||||
type:
|
||||
$ref: '#/components/schemas/Querybuildertypesv5QueryType'
|
||||
required:
|
||||
- type
|
||||
type: object
|
||||
Querybuildertypesv5QueryEnvelopeClickHouseSQL:
|
||||
properties:
|
||||
spec:
|
||||
@@ -7034,6 +7045,7 @@ components:
|
||||
Querybuildertypesv5QueryType:
|
||||
enum:
|
||||
- builder_query
|
||||
- builder_ai_query
|
||||
- builder_formula
|
||||
- builder_trace_operator
|
||||
- clickhouse_sql
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/metercollector"
|
||||
"github.com/SigNoz/signoz/pkg/modules/retention"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/licensetypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
@@ -153,7 +153,7 @@ func (provider *Provider) Collect(
|
||||
func buildOriginQuery(meterName string) (string, []any) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("toInt64(ifNull(min(unix_milli), 0))")
|
||||
sb.From(telemetrymeter.DBName + "." + telemetrymeter.SamplesTableName)
|
||||
sb.From(metertelemetryschema.DBName + "." + metertelemetryschema.SamplesTableName)
|
||||
sb.Where(sb.Equal("metric_name", meterName))
|
||||
return sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
}
|
||||
@@ -171,7 +171,7 @@ func buildQuery(meterName string, segment *retentiontypes.RetentionPolicySegment
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(selects...)
|
||||
sb.From(telemetrymeter.DBName + "." + telemetrymeter.SamplesTableName)
|
||||
sb.From(metertelemetryschema.DBName + "." + metertelemetryschema.SamplesTableName)
|
||||
sb.Where(
|
||||
sb.Equal("metric_name", meterName),
|
||||
sb.GTE("unix_milli", segment.StartMs),
|
||||
|
||||
@@ -8,16 +8,16 @@ import (
|
||||
sqlbuilder "github.com/huandu/go-sqlbuilder"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/metricreductionruletypes"
|
||||
)
|
||||
|
||||
var (
|
||||
reductionRulesTable = telemetrymetrics.DBName + "." + telemetrymetrics.ReductionRulesTableName
|
||||
metadataTable = telemetrymetrics.DBName + "." + telemetrymetrics.AttributesMetadataTableName
|
||||
bufferSeriesTable = telemetrymetrics.DBName + "." + telemetrymetrics.TimeseriesV4BufferTableName
|
||||
reductionRulesTable = metricstelemetryschema.DBName + "." + metricstelemetryschema.ReductionRulesTableName
|
||||
metadataTable = metricstelemetryschema.DBName + "." + metricstelemetryschema.AttributesMetadataTableName
|
||||
bufferSeriesTable = metricstelemetryschema.DBName + "." + metricstelemetryschema.TimeseriesV4BufferTableName
|
||||
)
|
||||
|
||||
const timeSeriesBucketMilli = int64(time.Hour / time.Millisecond)
|
||||
@@ -192,7 +192,7 @@ func (c *clickhouse) ingestedSeriesCount(ctx context.Context, metricNames []stri
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("metric_name", "uniq(fingerprint)")
|
||||
sb.From(telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName)
|
||||
sb.From(metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4BufferTableName)
|
||||
conds := []string{
|
||||
sb.In("metric_name", names...),
|
||||
sb.GE("unix_milli", startMs),
|
||||
@@ -229,8 +229,8 @@ func (c *clickhouse) ingestedSeriesCount(ctx context.Context, metricNames []stri
|
||||
// reduced sample tables.
|
||||
func (c *clickhouse) reducedSeriesCount(ctx context.Context, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (map[string]uint64, error) {
|
||||
out := make(map[string]uint64, len(metricNames))
|
||||
for _, table := range []string{telemetrymetrics.SamplesV4ReducedLastTableName, telemetrymetrics.SamplesV4ReducedSumTableName} {
|
||||
counts, err := c.reducedSeriesCountForTable(ctx, telemetrymetrics.DBName+"."+table, metricNames, effectiveFrom, startMs, endMs)
|
||||
for _, table := range []string{metricstelemetryschema.SamplesV4ReducedLastTableName, metricstelemetryschema.SamplesV4ReducedSumTableName} {
|
||||
counts, err := c.reducedSeriesCountForTable(ctx, metricstelemetryschema.DBName+"."+table, metricNames, effectiveFrom, startMs, endMs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -300,9 +300,9 @@ func (c *clickhouse) RankByVolume(ctx context.Context, metricNames []string, eff
|
||||
direction = "DESC"
|
||||
}
|
||||
|
||||
ingestedTable := telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName
|
||||
reducedLast := telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4ReducedLastTableName
|
||||
reducedSum := telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4ReducedSumTableName
|
||||
ingestedTable := metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4BufferTableName
|
||||
reducedLast := metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4ReducedLastTableName
|
||||
reducedSum := metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4ReducedSumTableName
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("base.metric_name AS metric_name", "ifNull(i.cnt, 0) AS ingested", "ifNull(d.cnt, 0) AS reduced")
|
||||
@@ -352,16 +352,16 @@ func (c *clickhouse) SampleVolumeByMetric(ctx context.Context, metricNames []str
|
||||
}
|
||||
ctx = c.withThreads(ctx)
|
||||
|
||||
ingested, err := c.countSamplesByMetric(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4BufferTableName, "count()", metricNames, effectiveFrom, startMs, endMs)
|
||||
ingested, err := c.countSamplesByMetric(ctx, metricstelemetryschema.DBName+"."+metricstelemetryschema.SamplesV4BufferTableName, "count()", metricNames, effectiveFrom, startMs, endMs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
last, err := c.countSamplesByMetric(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4ReducedLastTableName, "uniq(reduced_fingerprint, unix_milli)", metricNames, effectiveFrom, startMs, endMs)
|
||||
last, err := c.countSamplesByMetric(ctx, metricstelemetryschema.DBName+"."+metricstelemetryschema.SamplesV4ReducedLastTableName, "uniq(reduced_fingerprint, unix_milli)", metricNames, effectiveFrom, startMs, endMs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sum, err := c.countSamplesByMetric(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4ReducedSumTableName, "uniq(reduced_fingerprint, unix_milli)", metricNames, effectiveFrom, startMs, endMs)
|
||||
sum, err := c.countSamplesByMetric(ctx, metricstelemetryschema.DBName+"."+metricstelemetryschema.SamplesV4ReducedSumTableName, "uniq(reduced_fingerprint, unix_milli)", metricNames, effectiveFrom, startMs, endMs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -419,7 +419,7 @@ func (c *clickhouse) TotalVolume(ctx context.Context, startMs, endMs int64) (uin
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("uniq(fingerprint)", "count()")
|
||||
sb.From(telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName)
|
||||
sb.From(metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4BufferTableName)
|
||||
sb.Where(sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs))
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
@@ -470,7 +470,7 @@ func (c *clickhouse) SampleTimeseries(ctx context.Context, ruledMetrics []string
|
||||
func (c *clickhouse) totalSamplesByBucket(ctx context.Context, startMs, endMs int64) (map[int64]uint64, error) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(sampleBucketExpr, "count()")
|
||||
sb.From(telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName)
|
||||
sb.From(metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4BufferTableName)
|
||||
sb.Where(sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs))
|
||||
sb.GroupBy("bucket")
|
||||
|
||||
@@ -485,7 +485,7 @@ func (c *clickhouse) ruledIngestedSamplesByBucket(ctx context.Context, metricNam
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(sampleBucketExpr, "count()")
|
||||
sb.From(telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName)
|
||||
sb.From(metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4BufferTableName)
|
||||
conds := []string{sb.In("metric_name", names...), sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs)}
|
||||
if len(effectiveFrom) > 0 {
|
||||
conds = append(conds, strictEffectiveFrom(sb, metricNames, effectiveFrom))
|
||||
@@ -499,7 +499,7 @@ func (c *clickhouse) ruledIngestedSamplesByBucket(ctx context.Context, metricNam
|
||||
// reduced 60s rows are versioned by computed_at, so count distinct buckets.
|
||||
func (c *clickhouse) ruledRetainedSamplesByBucket(ctx context.Context, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (map[int64]uint64, error) {
|
||||
out := make(map[int64]uint64)
|
||||
for _, table := range []string{telemetrymetrics.SamplesV4ReducedLastTableName, telemetrymetrics.SamplesV4ReducedSumTableName} {
|
||||
for _, table := range []string{metricstelemetryschema.SamplesV4ReducedLastTableName, metricstelemetryschema.SamplesV4ReducedSumTableName} {
|
||||
names := make([]any, len(metricNames))
|
||||
for i, name := range metricNames {
|
||||
names[i] = name
|
||||
@@ -507,7 +507,7 @@ func (c *clickhouse) ruledRetainedSamplesByBucket(ctx context.Context, metricNam
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(sampleBucketExpr, "uniq(reduced_fingerprint, unix_milli)")
|
||||
sb.From(telemetrymetrics.DBName + "." + table)
|
||||
sb.From(metricstelemetryschema.DBName + "." + table)
|
||||
conds := []string{sb.In("metric_name", names...), sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs)}
|
||||
if len(effectiveFrom) > 0 {
|
||||
conds = append(conds, strictEffectiveFrom(sb, metricNames, effectiveFrom))
|
||||
|
||||
@@ -4298,6 +4298,18 @@ export interface Querybuildertypesv5QueryEnvelopeBuilderDTO {
|
||||
type: Querybuildertypesv5QueryEnvelopeBuilderDTOType;
|
||||
}
|
||||
|
||||
export enum Querybuildertypesv5QueryEnvelopeBuilderAIDTOType {
|
||||
builder_ai_query = 'builder_ai_query',
|
||||
}
|
||||
export interface Querybuildertypesv5QueryEnvelopeBuilderAIDTO {
|
||||
spec?: Querybuildertypesv5QueryBuilderQueryGithubComSigNozSignozPkgTypesQuerybuildertypesQuerybuildertypesv5TraceAggregationDTO;
|
||||
/**
|
||||
* @type string
|
||||
* @enum builder_ai_query
|
||||
*/
|
||||
type: Querybuildertypesv5QueryEnvelopeBuilderAIDTOType;
|
||||
}
|
||||
|
||||
export interface Querybuildertypesv5QueryBuilderFormulaDTO {
|
||||
/**
|
||||
* @type boolean
|
||||
@@ -4481,6 +4493,7 @@ export interface Querybuildertypesv5QueryEnvelopeClickHouseSQLDTO {
|
||||
|
||||
export type Querybuildertypesv5QueryEnvelopeDTO =
|
||||
| Querybuildertypesv5QueryEnvelopeBuilderDTO
|
||||
| Querybuildertypesv5QueryEnvelopeBuilderAIDTO
|
||||
| Querybuildertypesv5QueryEnvelopeFormulaDTO
|
||||
| Querybuildertypesv5QueryEnvelopeTraceOperatorDTO
|
||||
| Querybuildertypesv5QueryEnvelopePromQLDTO
|
||||
@@ -8284,6 +8297,7 @@ export interface Querybuildertypesv5QueryRangeResponseDTO {
|
||||
|
||||
export enum Querybuildertypesv5QueryTypeDTO {
|
||||
builder_query = 'builder_query',
|
||||
builder_ai_query = 'builder_ai_query',
|
||||
builder_formula = 'builder_formula',
|
||||
builder_trace_operator = 'builder_trace_operator',
|
||||
clickhouse_sql = 'clickhouse_sql',
|
||||
|
||||
@@ -5,14 +5,14 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
func (m *module) Collect(ctx context.Context, _ valuer.UUID) (map[string]any, error) {
|
||||
stats := make(map[string]any)
|
||||
|
||||
metadataTable := fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName)
|
||||
metadataTable := fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName)
|
||||
var (
|
||||
systemMetricCount uint64
|
||||
k8sMetricCount uint64
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -281,7 +281,7 @@ func (m *module) getPerGroupContainerStatusCounts(
|
||||
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
|
||||
|
||||
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
|
||||
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
|
||||
// Built once; identical across the two fps CTEs (buildFilterClause hits the
|
||||
// metadata store + parses the expression). AddWhereClause only reads it.
|
||||
@@ -310,7 +310,7 @@ func (m *module) getPerGroupContainerStatusCounts(
|
||||
)
|
||||
}
|
||||
stateFps.Select(stateFpsCols...)
|
||||
stateFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
stateFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
stateFps.Where(
|
||||
stateFps.E("metric_name", containerStatusStateMetricName),
|
||||
stateFps.GE("unix_milli", tsAdjustedStart),
|
||||
@@ -342,7 +342,7 @@ func (m *module) getPerGroupContainerStatusCounts(
|
||||
containerState.Select(containerStateCols...)
|
||||
containerState.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN state_fps AS fps ON samples.fingerprint = fps.fingerprint",
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
))
|
||||
containerState.Where(
|
||||
containerState.E("samples.metric_name", containerStatusStateMetricName),
|
||||
@@ -361,7 +361,7 @@ func (m *module) getPerGroupContainerStatusCounts(
|
||||
fmt.Sprintf("JSONExtractString(labels, %s) AS container_name", reasonFps.Var(containerNameAttrKey)),
|
||||
fmt.Sprintf("JSONExtractString(labels, %s) AS reason", reasonFps.Var(containerStatusReasonAttrKey)),
|
||||
)
|
||||
reasonFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
reasonFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
reasonFps.Where(
|
||||
reasonFps.E("metric_name", containerStatusReasonMetricName),
|
||||
reasonFps.GE("unix_milli", tsAdjustedStart),
|
||||
@@ -391,7 +391,7 @@ func (m *module) getPerGroupContainerStatusCounts(
|
||||
)
|
||||
reasonInner.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN reason_fps AS fps ON samples.fingerprint = fps.fingerprint",
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
))
|
||||
reasonInner.Where(
|
||||
reasonInner.E("samples.metric_name", containerStatusReasonMetricName),
|
||||
@@ -546,7 +546,7 @@ func (m *module) getPerGroupContainerRestartCounts(
|
||||
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
|
||||
|
||||
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
|
||||
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
|
||||
var (
|
||||
filterClause *sqlbuilder.WhereClause
|
||||
@@ -572,7 +572,7 @@ func (m *module) getPerGroupContainerRestartCounts(
|
||||
)
|
||||
}
|
||||
restartFps.Select(restartFpsCols...)
|
||||
restartFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
restartFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
restartFps.Where(
|
||||
restartFps.E("metric_name", containerRestartsMetricName),
|
||||
restartFps.GE("unix_milli", tsAdjustedStart),
|
||||
@@ -602,7 +602,7 @@ func (m *module) getPerGroupContainerRestartCounts(
|
||||
containerRestarts.Select(containerRestartsCols...)
|
||||
containerRestarts.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN restart_fps AS fps ON samples.fingerprint = fps.fingerprint",
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
))
|
||||
containerRestarts.Where(
|
||||
containerRestarts.E("samples.metric_name", containerRestartsMetricName),
|
||||
@@ -690,7 +690,7 @@ func (m *module) getPerGroupContainerReadyCounts(
|
||||
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
|
||||
|
||||
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
|
||||
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
|
||||
var (
|
||||
filterClause *sqlbuilder.WhereClause
|
||||
@@ -716,7 +716,7 @@ func (m *module) getPerGroupContainerReadyCounts(
|
||||
)
|
||||
}
|
||||
readyFps.Select(readyFpsCols...)
|
||||
readyFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
readyFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
readyFps.Where(
|
||||
readyFps.E("metric_name", containerReadyMetricName),
|
||||
readyFps.GE("unix_milli", tsAdjustedStart),
|
||||
@@ -746,7 +746,7 @@ func (m *module) getPerGroupContainerReadyCounts(
|
||||
containerReady.Select(containerReadyCols...)
|
||||
containerReady.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN ready_fps AS fps ON samples.fingerprint = fps.fingerprint",
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
))
|
||||
containerReady.Where(
|
||||
containerReady.E("samples.metric_name", containerReadyMetricName),
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
@@ -344,11 +344,11 @@ func alignedMetricWindow(startMs, endMs int64) (
|
||||
flooredEndMs = flooredEndMs - (flooredEndMs % (adjustStep * 1000))
|
||||
}
|
||||
|
||||
tsAdjustedStartMs, _, distributedTSTable, localTSTable := telemetrymetrics.WhichTSTableToUse(
|
||||
tsAdjustedStartMs, _, distributedTSTable, localTSTable := metricstelemetryschema.WhichTSTableToUse(
|
||||
samplesAdjustedStartMs, flooredEndMs, false, nil,
|
||||
)
|
||||
|
||||
distributedSamplesTable, localSamplesTable := telemetrymetrics.WhichSamplesTableToUse(
|
||||
distributedSamplesTable, localSamplesTable := metricstelemetryschema.WhichSamplesTableToUse(
|
||||
samplesAdjustedStartMs, flooredEndMs,
|
||||
metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil,
|
||||
)
|
||||
@@ -362,7 +362,7 @@ func alignedMetricWindow(startMs, endMs int64) (
|
||||
func (m *module) buildSamplesTblFingerprintSubQuery(metricNames []string, samplesTable string, flooredStart, flooredEnd uint64) *sqlbuilder.SelectBuilder {
|
||||
fpSB := sqlbuilder.NewSelectBuilder()
|
||||
fpSB.Select("DISTINCT fingerprint")
|
||||
fpSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, samplesTable))
|
||||
fpSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, samplesTable))
|
||||
fpSB.Where(
|
||||
fpSB.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
fpSB.GE("unix_milli", flooredStart),
|
||||
@@ -376,7 +376,7 @@ func (m *module) buildSamplesTblFingerprintSubQuery(metricNames []string, sample
|
||||
func (m *module) buildReducedSamplesTblFingerprintSubQuery(metricNames []string, flooredStart, flooredEnd uint64) *sqlbuilder.SelectBuilder {
|
||||
lastSB := sqlbuilder.NewSelectBuilder()
|
||||
lastSB.Select("reduced_fingerprint")
|
||||
lastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName))
|
||||
lastSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedLastTableName))
|
||||
lastSB.Where(
|
||||
lastSB.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
lastSB.GE("unix_milli", flooredStart),
|
||||
@@ -385,7 +385,7 @@ func (m *module) buildReducedSamplesTblFingerprintSubQuery(metricNames []string,
|
||||
|
||||
sumSB := sqlbuilder.NewSelectBuilder()
|
||||
sumSB.Select("reduced_fingerprint")
|
||||
sumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName))
|
||||
sumSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedSumTableName))
|
||||
sumSB.Where(
|
||||
sumSB.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
sumSB.GE("unix_milli", flooredStart),
|
||||
@@ -478,7 +478,7 @@ func (m *module) getEarliestMetricTime(ctx context.Context, metricNames []string
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("min(first_reported_unix_milli) AS min_first_reported")
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
|
||||
sb.Where(sb.In("metric_name", sqlbuilder.List(metricNames)))
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
@@ -504,7 +504,7 @@ func (m *module) getMetricsExistence(ctx context.Context, metricNames []string)
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("metric_name", "count(*) AS cnt")
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
|
||||
sb.Where(sb.In("metric_name", sqlbuilder.List(metricNames)))
|
||||
sb.GroupBy("metric_name")
|
||||
|
||||
@@ -549,7 +549,7 @@ func (m *module) getAttributesExistence(ctx context.Context, metricNames, attrNa
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("attr_name", "count(*) AS cnt")
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
|
||||
sb.Where(
|
||||
sb.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
sb.In("attr_name", sqlbuilder.List(attrNames)),
|
||||
@@ -656,7 +656,7 @@ func (m *module) getMetadata(
|
||||
|
||||
rawSrc := sqlbuilder.NewSelectBuilder()
|
||||
rawSrc.Select("labels", "unix_milli")
|
||||
rawSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTableName))
|
||||
rawSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTableName))
|
||||
rawSrc.Where(
|
||||
rawSrc.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
rawSrc.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -669,7 +669,7 @@ func (m *module) getMetadata(
|
||||
|
||||
reducedSrc := sqlbuilder.NewSelectBuilder()
|
||||
reducedSrc.Select("labels", "unix_milli")
|
||||
reducedSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
|
||||
reducedSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
|
||||
reducedSrc.Where(
|
||||
reducedSrc.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
reducedSrc.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -683,7 +683,7 @@ func (m *module) getMetadata(
|
||||
// Inner query reads over the union of raw + reduced series.
|
||||
innerSB.From(innerSB.BuilderAs(sqlbuilder.UnionAll(rawSrc, reducedSrc), "series"))
|
||||
} else {
|
||||
innerSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTableName))
|
||||
innerSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTableName))
|
||||
innerSB.Where(
|
||||
innerSB.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
innerSB.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -870,7 +870,7 @@ func (m *module) getPerGroupDistinctCounts(
|
||||
|
||||
rawSrc := sqlbuilder.NewSelectBuilder()
|
||||
rawSrc.Select("labels")
|
||||
rawSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTimeSeriesTbl))
|
||||
rawSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTimeSeriesTbl))
|
||||
rawSrc.Where(
|
||||
rawSrc.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
rawSrc.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -883,7 +883,7 @@ func (m *module) getPerGroupDistinctCounts(
|
||||
|
||||
reducedSrc := sqlbuilder.NewSelectBuilder()
|
||||
reducedSrc.Select("labels")
|
||||
reducedSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
|
||||
reducedSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
|
||||
reducedSrc.Where(
|
||||
reducedSrc.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
reducedSrc.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -896,7 +896,7 @@ func (m *module) getPerGroupDistinctCounts(
|
||||
|
||||
sb.From(sb.BuilderAs(sqlbuilder.UnionAll(rawSrc, reducedSrc), "series"))
|
||||
} else {
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTimeSeriesTbl))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTimeSeriesTbl))
|
||||
sb.Where(
|
||||
sb.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
sb.GE("unix_milli", tsAdjustedStartMs),
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -72,7 +72,7 @@ func (m *module) getPerGroupHostStatusCounts(
|
||||
|
||||
rawSrc := sqlbuilder.NewSelectBuilder()
|
||||
rawSrc.Select("labels")
|
||||
rawSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTimeSeriesTableName))
|
||||
rawSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTimeSeriesTableName))
|
||||
rawSrc.Where(
|
||||
rawSrc.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
rawSrc.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -85,7 +85,7 @@ func (m *module) getPerGroupHostStatusCounts(
|
||||
|
||||
reducedSrc := sqlbuilder.NewSelectBuilder()
|
||||
reducedSrc.Select("labels")
|
||||
reducedSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
|
||||
reducedSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
|
||||
reducedSrc.Where(
|
||||
reducedSrc.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
reducedSrc.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -101,7 +101,7 @@ func (m *module) getPerGroupHostStatusCounts(
|
||||
|
||||
fpSB := m.buildSamplesTblFingerprintSubQuery(metricNames, localSamplesTable, samplesStartMs, flooredEndMs)
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTimeSeriesTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTimeSeriesTableName))
|
||||
sb.Where(
|
||||
sb.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
sb.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -357,7 +357,7 @@ func (m *module) getActiveHostsQuery(metricNames []string, hostNameAttr string,
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Distinct()
|
||||
sb.Select("attr_string_value")
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
|
||||
sb.Where(
|
||||
sb.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
sb.E("attr_name", hostNameAttr),
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/modules/inframonitoring"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
|
||||
@@ -40,8 +40,8 @@ func NewModule(
|
||||
providerSettings factory.ProviderSettings,
|
||||
cfg inframonitoring.Config,
|
||||
) inframonitoring.Module {
|
||||
fieldMapper := telemetrymetrics.NewFieldMapper()
|
||||
condBuilder := telemetrymetrics.NewConditionBuilder(fieldMapper)
|
||||
fieldMapper := metricstelemetryschema.NewFieldMapper()
|
||||
condBuilder := metricstelemetryschema.NewConditionBuilder(fieldMapper)
|
||||
return &module{
|
||||
telemetryStore: telemetryStore,
|
||||
telemetryMetadataStore: telemetryMetadataStore,
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -186,7 +186,7 @@ func (m *module) getPerGroupNodeConditionCounts(
|
||||
|
||||
// Step-floor bounds + resolve tables in one shot to match QB v5 querier.
|
||||
samplesStartMs, flooredEndMs, tsAdjustedStartMs, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
|
||||
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
|
||||
// ----- timeSeriesFPs -----
|
||||
timeSeriesFPs := sqlbuilder.NewSelectBuilder()
|
||||
@@ -200,7 +200,7 @@ func (m *module) getPerGroupNodeConditionCounts(
|
||||
)
|
||||
}
|
||||
timeSeriesFPs.Select(timeSeriesFPsSelectCols...)
|
||||
timeSeriesFPs.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
timeSeriesFPs.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
timeSeriesFPs.Where(
|
||||
timeSeriesFPs.E("metric_name", nodeConditionMetricName),
|
||||
timeSeriesFPs.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -237,7 +237,7 @@ func (m *module) getPerGroupNodeConditionCounts(
|
||||
latestConditionPerNode.Select(latestConditionPerNodeSelectCols...)
|
||||
latestConditionPerNode.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN time_series_fps AS tsfp ON samples.fingerprint = tsfp.fingerprint",
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
))
|
||||
latestConditionPerNode.Where(
|
||||
latestConditionPerNode.E("samples.metric_name", nodeConditionMetricName),
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -284,7 +284,7 @@ func (m *module) getPerGroupPodStatusCounts(
|
||||
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
|
||||
|
||||
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
|
||||
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
|
||||
// Build the merged filter clause once; it's identical across the three fps
|
||||
// CTEs, and buildFilterClause hits the metadata store + parses the
|
||||
@@ -313,7 +313,7 @@ func (m *module) getPerGroupPodStatusCounts(
|
||||
)
|
||||
}
|
||||
phaseFps.Select(phaseFpsCols...)
|
||||
phaseFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
phaseFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
phaseFps.Where(
|
||||
phaseFps.E("metric_name", podPhaseMetricName),
|
||||
phaseFps.GE("unix_milli", tsAdjustedStart),
|
||||
@@ -340,7 +340,7 @@ func (m *module) getPerGroupPodStatusCounts(
|
||||
phasePerPod.Select(phasePerPodCols...)
|
||||
phasePerPod.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN phase_fps AS fps ON samples.fingerprint = fps.fingerprint",
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
))
|
||||
phasePerPod.Where(
|
||||
phasePerPod.E("samples.metric_name", podPhaseMetricName),
|
||||
@@ -357,7 +357,7 @@ func (m *module) getPerGroupPodStatusCounts(
|
||||
"fingerprint",
|
||||
fmt.Sprintf("JSONExtractString(labels, %s) AS pod_uid", podReasonFps.Var(podUIDAttrKey)),
|
||||
)
|
||||
podReasonFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
podReasonFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
podReasonFps.Where(
|
||||
podReasonFps.E("metric_name", podStatusReasonMetricName),
|
||||
podReasonFps.GE("unix_milli", tsAdjustedStart),
|
||||
@@ -377,7 +377,7 @@ func (m *module) getPerGroupPodStatusCounts(
|
||||
)
|
||||
podReasonPerPod.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN pod_reason_fps AS fps ON samples.fingerprint = fps.fingerprint",
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
))
|
||||
podReasonPerPod.Where(
|
||||
podReasonPerPod.E("samples.metric_name", podStatusReasonMetricName),
|
||||
@@ -396,7 +396,7 @@ func (m *module) getPerGroupPodStatusCounts(
|
||||
fmt.Sprintf("JSONExtractString(labels, %s) AS container_name", containerReasonFps.Var(containerNameAttrKey)),
|
||||
fmt.Sprintf("JSONExtractString(labels, %s) AS reason", containerReasonFps.Var(containerStatusReasonAttrKey)),
|
||||
)
|
||||
containerReasonFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
containerReasonFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
containerReasonFps.Where(
|
||||
containerReasonFps.E("metric_name", containerStatusReasonMetricName),
|
||||
containerReasonFps.GE("unix_milli", tsAdjustedStart),
|
||||
@@ -432,7 +432,7 @@ func (m *module) getPerGroupPodStatusCounts(
|
||||
)
|
||||
containerInner.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN container_reason_fps AS fps ON samples.fingerprint = fps.fingerprint",
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
))
|
||||
containerInner.Where(
|
||||
containerInner.E("samples.metric_name", containerStatusReasonMetricName),
|
||||
@@ -611,7 +611,7 @@ func (m *module) getPerGroupPodRestartCounts(
|
||||
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
|
||||
|
||||
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
|
||||
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
|
||||
var (
|
||||
filterClause *sqlbuilder.WhereClause
|
||||
@@ -637,7 +637,7 @@ func (m *module) getPerGroupPodRestartCounts(
|
||||
)
|
||||
}
|
||||
restartFps.Select(restartFpsCols...)
|
||||
restartFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
restartFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
restartFps.Where(
|
||||
restartFps.E("metric_name", containerRestartsMetricName),
|
||||
restartFps.GE("unix_milli", tsAdjustedStart),
|
||||
@@ -667,7 +667,7 @@ func (m *module) getPerGroupPodRestartCounts(
|
||||
containerRestarts.Select(containerRestartsCols...)
|
||||
containerRestarts.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN restart_fps AS fps ON samples.fingerprint = fps.fingerprint",
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
))
|
||||
containerRestarts.Where(
|
||||
containerRestarts.E("samples.metric_name", containerRestartsMetricName),
|
||||
|
||||
@@ -213,18 +213,18 @@ func (module *module) discoverModels(ctx context.Context, orgID valuer.UUID) ([]
|
||||
Spec: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
|
||||
Name: "A",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
Filter: &qbtypes.Filter{Expression: fmt.Sprintf("%s EXISTS", llmpricingruletypes.GenAIRequestModel)},
|
||||
Filter: &qbtypes.Filter{Expression: fmt.Sprintf("%s EXISTS", telemetrytypes.GenAIRequestModel)},
|
||||
Aggregations: []qbtypes.TraceAggregation{
|
||||
{Expression: "count()", Alias: "spanCount"},
|
||||
},
|
||||
GroupBy: []qbtypes.GroupByKey{
|
||||
{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: llmpricingruletypes.GenAIRequestModel,
|
||||
Name: telemetrytypes.GenAIRequestModel,
|
||||
FieldContext: telemetrytypes.FieldContextSpan,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}},
|
||||
{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: llmpricingruletypes.GenAIProviderName,
|
||||
Name: telemetrytypes.GenAIProviderName,
|
||||
FieldContext: telemetrytypes.FieldContextSpan,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}},
|
||||
@@ -254,9 +254,9 @@ func (module *module) discoverModels(ctx context.Context, orgID valuer.UUID) ([]
|
||||
switch c.Type {
|
||||
case qbtypes.ColumnTypeGroup:
|
||||
switch c.Name {
|
||||
case llmpricingruletypes.GenAIRequestModel:
|
||||
case telemetrytypes.GenAIRequestModel:
|
||||
modelIdx = i
|
||||
case llmpricingruletypes.GenAIProviderName:
|
||||
case telemetrytypes.GenAIProviderName:
|
||||
providerIdx = i
|
||||
}
|
||||
case qbtypes.ColumnTypeAggregation:
|
||||
|
||||
@@ -19,8 +19,8 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/modules/dashboard"
|
||||
"github.com/SigNoz/signoz/pkg/modules/metricsexplorer"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
@@ -49,8 +49,8 @@ type module struct {
|
||||
|
||||
// NewModule constructs the metrics module with the provided dependencies.
|
||||
func NewModule(ts telemetrystore.TelemetryStore, telemetryMetadataStore telemetrytypes.MetadataStore, cache cache.Cache, ruleStore ruletypes.RuleStore, dashboardModule dashboard.Module, fl flagger.Flagger, providerSettings factory.ProviderSettings, cfg metricsexplorer.Config) metricsexplorer.Module {
|
||||
fieldMapper := telemetrymetrics.NewFieldMapper()
|
||||
condBuilder := telemetrymetrics.NewConditionBuilder(fieldMapper)
|
||||
fieldMapper := metricstelemetryschema.NewFieldMapper()
|
||||
condBuilder := metricstelemetryschema.NewConditionBuilder(fieldMapper)
|
||||
return &module{
|
||||
telemetryStore: ts,
|
||||
fieldMapper: fieldMapper,
|
||||
@@ -89,7 +89,7 @@ func (m *module) listMeterMetrics(ctx context.Context, params *metricsexplorerty
|
||||
"argMax(temporality, unix_milli) AS temporality",
|
||||
"argMax(is_monotonic, unix_milli) AS is_monotonic",
|
||||
)
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymeter.DBName, telemetrymeter.SamplesTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metertelemetryschema.DBName, metertelemetryschema.SamplesTableName))
|
||||
|
||||
if params.Start != nil && params.End != nil {
|
||||
sb.Where(sb.Between("unix_milli", *params.Start, *params.End))
|
||||
@@ -160,11 +160,11 @@ func (m *module) listMetrics(ctx context.Context, orgID valuer.UUID, params *met
|
||||
hasRange := params.Start != nil && params.End != nil
|
||||
if hasRange {
|
||||
var distributedTsTable string
|
||||
rangeStart, rangeEnd, distributedTsTable, _ = telemetrymetrics.WhichTSTableToUse(uint64(*params.Start), uint64(*params.End), false, nil)
|
||||
rawSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
|
||||
rangeStart, rangeEnd, distributedTsTable, _ = metricstelemetryschema.WhichTSTableToUse(uint64(*params.Start), uint64(*params.End), false, nil)
|
||||
rawSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTsTable))
|
||||
rawSB.Where(rawSB.Between("unix_milli", rangeStart, rangeEnd))
|
||||
} else {
|
||||
rawSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV41weekTableName))
|
||||
rawSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV41weekTableName))
|
||||
}
|
||||
|
||||
applyListFilters(rawSB)
|
||||
@@ -174,7 +174,7 @@ func (m *module) listMetrics(ctx context.Context, orgID valuer.UUID, params *met
|
||||
if reductionEnabled {
|
||||
reducedSB := sqlbuilder.NewSelectBuilder()
|
||||
reducedSB.Select("DISTINCT metric_name")
|
||||
reducedSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
|
||||
reducedSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
|
||||
if hasRange {
|
||||
reducedSB.Where(reducedSB.Between("unix_milli", rangeStart, rangeEnd))
|
||||
}
|
||||
@@ -512,7 +512,7 @@ func (m *module) CheckMetricExists(ctx context.Context, orgID valuer.UUID, metri
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("count(*) > 0 as metricExists")
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
|
||||
sb.Where(sb.E("metric_name", metricName))
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
@@ -534,7 +534,7 @@ func (m *module) HasNonSigNozMetrics(ctx context.Context) (bool, error) {
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("count(*) > 0")
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV41weekTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV41weekTableName))
|
||||
sb.Where("metric_name NOT LIKE 'signoz_%'")
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
@@ -570,10 +570,10 @@ func (m *module) InspectMetrics(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tsStart, _, tsTable, _ := telemetrymetrics.WhichTSTableToUse(start, end, false, nil)
|
||||
tsStart, _, tsTable, _ := metricstelemetryschema.WhichTSTableToUse(start, end, false, nil)
|
||||
tsSb := sqlbuilder.NewSelectBuilder()
|
||||
tsSb.Select("fingerprint", "labels")
|
||||
tsSb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, tsTable))
|
||||
tsSb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, tsTable))
|
||||
tsSb.Where(
|
||||
tsSb.E("metric_name", req.MetricName),
|
||||
tsSb.GE("unix_milli", tsStart),
|
||||
@@ -625,7 +625,7 @@ func (m *module) InspectMetrics(
|
||||
|
||||
samplesSb := sqlbuilder.NewSelectBuilder()
|
||||
samplesSb.Select("fingerprint", "unix_milli", "value")
|
||||
samplesSb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4TableName))
|
||||
samplesSb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4TableName))
|
||||
samplesSb.Where(
|
||||
samplesSb.In("fingerprint", fingerprints...),
|
||||
samplesSb.E("metric_name", req.MetricName),
|
||||
@@ -717,7 +717,7 @@ func (m *module) fetchUpdatedMetadata(ctx context.Context, orgID valuer.UUID, me
|
||||
"argMax(temporality, created_at) AS temporality",
|
||||
"argMax(is_monotonic, created_at) AS is_monotonic",
|
||||
)
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.UpdatedMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.UpdatedMetadataTableName))
|
||||
sb.Where(sb.In("metric_name", args...))
|
||||
sb.GroupBy("metric_name")
|
||||
|
||||
@@ -777,7 +777,7 @@ func (m *module) fetchTimeseriesMetadata(ctx context.Context, orgID valuer.UUID,
|
||||
"anyLast(temporality) AS temporality",
|
||||
"argMax(is_monotonic, unix_milli) AS is_monotonic",
|
||||
)
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4TableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4TableName))
|
||||
sb.Where(sb.In("metric_name", args...))
|
||||
sb.GroupBy("metric_name")
|
||||
|
||||
@@ -891,7 +891,7 @@ func (m *module) checkForLabelInMetric(ctx context.Context, metricName string, l
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("count(*) > 0 AS has_label")
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
|
||||
sb.Where(sb.E("metric_name", metricName))
|
||||
sb.Where(sb.E("attr_name", label))
|
||||
sb.Limit(1)
|
||||
@@ -914,7 +914,7 @@ func (m *module) insertMetricsMetadata(ctx context.Context, orgID valuer.UUID, r
|
||||
createdAt := time.Now().UnixMilli()
|
||||
|
||||
ib := sqlbuilder.NewInsertBuilder()
|
||||
ib.InsertInto(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.UpdatedMetadataTableName))
|
||||
ib.InsertInto(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.UpdatedMetadataTableName))
|
||||
ib.Cols("metric_name", "temporality", "is_monotonic", "type", "description", "unit", "created_at")
|
||||
ib.Values(
|
||||
req.MetricName,
|
||||
@@ -1016,9 +1016,9 @@ func (m *module) fetchMetricsStatsWithSamples(
|
||||
}
|
||||
}
|
||||
|
||||
start, end, distributedTsTable, localTsTable := telemetrymetrics.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
|
||||
distributedSamplesTable, _ := telemetrymetrics.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil)
|
||||
countExp := telemetrymetrics.CountExpressionForSamplesTable(distributedSamplesTable)
|
||||
start, end, distributedTsTable, localTsTable := metricstelemetryschema.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
|
||||
distributedSamplesTable, _ := metricstelemetryschema.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil)
|
||||
countExp := metricstelemetryschema.CountExpressionForSamplesTable(distributedSamplesTable)
|
||||
|
||||
// Timeseries counts per metric
|
||||
tsSB := sqlbuilder.NewSelectBuilder()
|
||||
@@ -1026,7 +1026,7 @@ func (m *module) fetchMetricsStatsWithSamples(
|
||||
"metric_name",
|
||||
"uniq(fingerprint) AS timeseries",
|
||||
)
|
||||
tsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
|
||||
tsSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTsTable))
|
||||
tsSB.Where(tsSB.Between("unix_milli", start, end))
|
||||
tsSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
if filterWhereClause != nil {
|
||||
@@ -1040,7 +1040,7 @@ func (m *module) fetchMetricsStatsWithSamples(
|
||||
"metric_name",
|
||||
fmt.Sprintf("%s AS samples", countExp),
|
||||
)
|
||||
samplesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedSamplesTable))
|
||||
samplesSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedSamplesTable))
|
||||
samplesSB.Where(samplesSB.Between("unix_milli", req.Start, req.End))
|
||||
samplesSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
|
||||
@@ -1053,7 +1053,7 @@ func (m *module) fetchMetricsStatsWithSamples(
|
||||
if filterWhereClause != nil {
|
||||
fingerprintSB := sqlbuilder.NewSelectBuilder()
|
||||
fingerprintSB.Select("fingerprint")
|
||||
fingerprintSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTsTable))
|
||||
fingerprintSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTsTable))
|
||||
fingerprintSB.Where(fingerprintSB.Between("unix_milli", start, end))
|
||||
fingerprintSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
fingerprintSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause))
|
||||
@@ -1064,7 +1064,7 @@ func (m *module) fetchMetricsStatsWithSamples(
|
||||
} else {
|
||||
metricNamesSB := sqlbuilder.NewSelectBuilder()
|
||||
metricNamesSB.Select("DISTINCT metric_name")
|
||||
metricNamesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTsTable))
|
||||
metricNamesSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTsTable))
|
||||
metricNamesSB.Where(metricNamesSB.Between("unix_milli", start, end))
|
||||
metricNamesSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
|
||||
@@ -1090,18 +1090,18 @@ func (m *module) fetchMetricsStatsWithSamples(
|
||||
// the data lands either of the _reduced_last/_reduced_sum tables
|
||||
reducedLastSB := sqlbuilder.NewSelectBuilder()
|
||||
reducedLastSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt")
|
||||
reducedLastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName))
|
||||
reducedLastSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedLastTableName))
|
||||
reducedLastSB.Where(reducedLastSB.Between("unix_milli", req.Start, req.End))
|
||||
reducedLastSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
|
||||
reducedSumSB := sqlbuilder.NewSelectBuilder()
|
||||
reducedSumSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt")
|
||||
reducedSumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName))
|
||||
reducedSumSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedSumTableName))
|
||||
reducedSumSB.Where(reducedSumSB.Between("unix_milli", req.Start, req.End))
|
||||
reducedSumSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
|
||||
// separate query for reduced series counts
|
||||
reducedTsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
|
||||
reducedTsSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
|
||||
reducedTsSB.Where(reducedTsSB.Between("unix_milli", start, end))
|
||||
reducedTsSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
reducedTsSB.GroupBy("metric_name")
|
||||
@@ -1112,7 +1112,7 @@ func (m *module) fetchMetricsStatsWithSamples(
|
||||
// samples uses a separate cte with local table
|
||||
reducedFpSB := sqlbuilder.NewSelectBuilder()
|
||||
reducedFpSB.Select("fingerprint")
|
||||
reducedFpSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedLocalTableName))
|
||||
reducedFpSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedLocalTableName))
|
||||
reducedFpSB.Where(reducedFpSB.Between("unix_milli", start, end))
|
||||
reducedFpSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
reducedFpSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause))
|
||||
@@ -1220,11 +1220,11 @@ func (m *module) computeTimeseriesTreemap(ctx context.Context, orgID valuer.UUID
|
||||
}
|
||||
}
|
||||
|
||||
start, end, distributedTsTable, _ := telemetrymetrics.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
|
||||
start, end, distributedTsTable, _ := metricstelemetryschema.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
|
||||
|
||||
totalTSBuilder := sqlbuilder.NewSelectBuilder()
|
||||
totalTSBuilder.Select("uniq(fingerprint) AS total_time_series")
|
||||
totalTSBuilder.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
|
||||
totalTSBuilder.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTsTable))
|
||||
totalTSBuilder.Where(totalTSBuilder.Between("unix_milli", start, end))
|
||||
|
||||
metricsSB := sqlbuilder.NewSelectBuilder()
|
||||
@@ -1232,7 +1232,7 @@ func (m *module) computeTimeseriesTreemap(ctx context.Context, orgID valuer.UUID
|
||||
"metric_name",
|
||||
"uniq(fingerprint) AS total_value",
|
||||
)
|
||||
metricsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
|
||||
metricsSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTsTable))
|
||||
metricsSB.Where(metricsSB.Between("unix_milli", start, end))
|
||||
metricsSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
if filterWhereClause != nil {
|
||||
@@ -1244,7 +1244,7 @@ func (m *module) computeTimeseriesTreemap(ctx context.Context, orgID valuer.UUID
|
||||
if reductionEnabled {
|
||||
reducedTotalTSBuilder := sqlbuilder.NewSelectBuilder()
|
||||
reducedTotalTSBuilder.Select("uniq(fingerprint) AS total_time_series")
|
||||
reducedTotalTSBuilder.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
|
||||
reducedTotalTSBuilder.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
|
||||
reducedTotalTSBuilder.Where(reducedTotalTSBuilder.Between("unix_milli", start, end))
|
||||
|
||||
reducedMetricsSB := sqlbuilder.NewSelectBuilder()
|
||||
@@ -1252,7 +1252,7 @@ func (m *module) computeTimeseriesTreemap(ctx context.Context, orgID valuer.UUID
|
||||
"metric_name",
|
||||
"uniq(fingerprint) AS total_value",
|
||||
)
|
||||
reducedMetricsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
|
||||
reducedMetricsSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
|
||||
reducedMetricsSB.Where(reducedMetricsSB.Between("unix_milli", start, end))
|
||||
reducedMetricsSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
if filterWhereClause != nil {
|
||||
@@ -1338,15 +1338,15 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
|
||||
}
|
||||
}
|
||||
|
||||
start, end, distributedTsTable, localTsTable := telemetrymetrics.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
|
||||
distributedSamplesTable, _ := telemetrymetrics.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil)
|
||||
countExp := telemetrymetrics.CountExpressionForSamplesTable(distributedSamplesTable)
|
||||
start, end, distributedTsTable, localTsTable := metricstelemetryschema.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
|
||||
distributedSamplesTable, _ := metricstelemetryschema.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil)
|
||||
countExp := metricstelemetryschema.CountExpressionForSamplesTable(distributedSamplesTable)
|
||||
|
||||
candidateLimit := req.Limit + 50
|
||||
|
||||
metricCandidatesSB := sqlbuilder.NewSelectBuilder()
|
||||
metricCandidatesSB.Select("metric_name")
|
||||
metricCandidatesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
|
||||
metricCandidatesSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTsTable))
|
||||
metricCandidatesSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
metricCandidatesSB.Where(metricCandidatesSB.Between("unix_milli", start, end))
|
||||
if filterWhereClause != nil {
|
||||
@@ -1364,7 +1364,7 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
|
||||
if reductionEnabled {
|
||||
reducedCandidatesSB = sqlbuilder.NewSelectBuilder()
|
||||
reducedCandidatesSB.Select("metric_name")
|
||||
reducedCandidatesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
|
||||
reducedCandidatesSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
|
||||
reducedCandidatesSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
reducedCandidatesSB.Where(reducedCandidatesSB.Between("unix_milli", start, end))
|
||||
if filterWhereClause != nil {
|
||||
@@ -1379,7 +1379,7 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
|
||||
|
||||
totalSamplesSB := sqlbuilder.NewSelectBuilder()
|
||||
totalSamplesSB.Select(fmt.Sprintf("%s AS total_samples", countExp))
|
||||
totalSamplesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedSamplesTable))
|
||||
totalSamplesSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedSamplesTable))
|
||||
totalSamplesSB.Where(totalSamplesSB.Between("unix_milli", req.Start, req.End))
|
||||
|
||||
sampleCountsSB := sqlbuilder.NewSelectBuilder()
|
||||
@@ -1387,7 +1387,7 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
|
||||
"metric_name",
|
||||
fmt.Sprintf("%s AS samples", countExp),
|
||||
)
|
||||
sampleCountsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedSamplesTable))
|
||||
sampleCountsSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedSamplesTable))
|
||||
sampleCountsSB.Where(sampleCountsSB.Between("unix_milli", req.Start, req.End))
|
||||
sampleCountsSB.Where("metric_name GLOBAL IN (SELECT metric_name FROM __metric_candidates)")
|
||||
|
||||
@@ -1397,13 +1397,13 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
|
||||
if reductionEnabled {
|
||||
reducedLastSB = sqlbuilder.NewSelectBuilder()
|
||||
reducedLastSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt")
|
||||
reducedLastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName))
|
||||
reducedLastSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedLastTableName))
|
||||
reducedLastSB.Where(reducedLastSB.Between("unix_milli", req.Start, req.End))
|
||||
reducedLastSB.Where("metric_name GLOBAL IN (SELECT metric_name FROM __reduced_metric_candidates)")
|
||||
|
||||
reducedSumSB = sqlbuilder.NewSelectBuilder()
|
||||
reducedSumSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt")
|
||||
reducedSumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName))
|
||||
reducedSumSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedSumTableName))
|
||||
reducedSumSB.Where(reducedSumSB.Between("unix_milli", req.Start, req.End))
|
||||
reducedSumSB.Where("metric_name GLOBAL IN (SELECT metric_name FROM __reduced_metric_candidates)")
|
||||
}
|
||||
@@ -1411,7 +1411,7 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
|
||||
if filterWhereClause != nil {
|
||||
fingerprintSB := sqlbuilder.NewSelectBuilder()
|
||||
fingerprintSB.Select("fingerprint")
|
||||
fingerprintSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTsTable))
|
||||
fingerprintSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTsTable))
|
||||
fingerprintSB.Where(fingerprintSB.Between("unix_milli", start, end))
|
||||
fingerprintSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
fingerprintSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause))
|
||||
@@ -1423,7 +1423,7 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
|
||||
if reductionEnabled {
|
||||
reducedFingerprintSB := sqlbuilder.NewSelectBuilder()
|
||||
reducedFingerprintSB.Select("fingerprint")
|
||||
reducedFingerprintSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedLocalTableName))
|
||||
reducedFingerprintSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedLocalTableName))
|
||||
reducedFingerprintSB.Where(reducedFingerprintSB.Between("unix_milli", start, end))
|
||||
reducedFingerprintSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
reducedFingerprintSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause))
|
||||
@@ -1457,12 +1457,12 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
|
||||
// Grand total includes reduced sample volume so percentages stay consistent.
|
||||
reducedTotalLastSB := sqlbuilder.NewSelectBuilder()
|
||||
reducedTotalLastSB.Select("uniq(reduced_fingerprint, unix_milli) AS cnt")
|
||||
reducedTotalLastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName))
|
||||
reducedTotalLastSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedLastTableName))
|
||||
reducedTotalLastSB.Where(reducedTotalLastSB.Between("unix_milli", req.Start, req.End))
|
||||
|
||||
reducedTotalSumSB := sqlbuilder.NewSelectBuilder()
|
||||
reducedTotalSumSB.Select("uniq(reduced_fingerprint, unix_milli) AS cnt")
|
||||
reducedTotalSumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName))
|
||||
reducedTotalSumSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedSumTableName))
|
||||
reducedTotalSumSB.Where(reducedTotalSumSB.Between("unix_milli", req.Start, req.End))
|
||||
|
||||
reducedTotalSamplesSB := sqlbuilder.NewSelectBuilder()
|
||||
@@ -1552,7 +1552,7 @@ func (m *module) getMetricDataPoints(ctx context.Context, metricName string) (ui
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("sum(count) AS data_points")
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4Agg30mTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4Agg30mTableName))
|
||||
sb.Where(sb.E("metric_name", metricName))
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
@@ -1574,7 +1574,7 @@ func (m *module) getMetricLastReceived(ctx context.Context, metricName string) (
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("MAX(last_reported_unix_milli) AS last_received_time")
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
|
||||
sb.Where(sb.E("metric_name", metricName))
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
@@ -1599,7 +1599,7 @@ func (m *module) getTotalTimeSeriesForMetricName(ctx context.Context, metricName
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("uniq(fingerprint) AS time_series_count")
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV41weekTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV41weekTableName))
|
||||
sb.Where(sb.E("metric_name", metricName))
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
@@ -1623,7 +1623,7 @@ func (m *module) getActiveTimeSeriesForMetricName(ctx context.Context, metricNam
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("uniq(fingerprint) AS active_time_series")
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4TableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4TableName))
|
||||
sb.Where(sb.E("metric_name", metricName))
|
||||
sb.Where(sb.GTE("unix_milli", milli))
|
||||
|
||||
@@ -1649,7 +1649,7 @@ func (m *module) fetchMetricAttributes(ctx context.Context, metricName string, s
|
||||
"groupUniqArray(1000)(attr_string_value) AS values",
|
||||
"uniq(attr_string_value) AS valueCount",
|
||||
)
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
|
||||
sb.Where(sb.E("metric_name", metricName))
|
||||
sb.Where("NOT startsWith(attr_name, '__')")
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/modules/promote"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
@@ -52,7 +52,7 @@ func (m *module) ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetype
|
||||
|
||||
response := []promotetypes.PromotePath{}
|
||||
for _, path := range promotedPaths {
|
||||
fullPath := telemetrylogs.BodyPromotedColumnPrefix + path
|
||||
fullPath := logstelemetryschema.BodyPromotedColumnPrefix + path
|
||||
path = telemetrytypes.BodyJSONStringSearchPrefix + path
|
||||
item := promotetypes.PromotePath{
|
||||
Path: path,
|
||||
@@ -68,7 +68,7 @@ func (m *module) ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetype
|
||||
|
||||
// add the paths that are not promoted but have indexes
|
||||
for path, indexes := range aggr {
|
||||
path := strings.TrimPrefix(path, telemetrylogs.BodyV2ColumnPrefix)
|
||||
path := strings.TrimPrefix(path, logstelemetryschema.BodyV2ColumnPrefix)
|
||||
path = telemetrytypes.BodyJSONStringSearchPrefix + path
|
||||
response = append(response, promotetypes.PromotePath{
|
||||
Path: path,
|
||||
@@ -108,8 +108,8 @@ func (m *module) createIndexes(ctx context.Context, indexes []schemamigrator.Ind
|
||||
|
||||
for _, index := range indexes {
|
||||
alterStmt := schemamigrator.AlterTableAddIndex{
|
||||
Database: telemetrylogs.DBName,
|
||||
Table: telemetrylogs.LogsV2LocalTableName,
|
||||
Database: logstelemetryschema.DBName,
|
||||
Table: logstelemetryschema.LogsV2LocalTableName,
|
||||
Index: index,
|
||||
}
|
||||
op := alterStmt.OnCluster(m.telemetryStore.Cluster())
|
||||
@@ -153,10 +153,10 @@ func (m *module) PromoteAndIndexPaths(
|
||||
}
|
||||
}
|
||||
if len(it.Indexes) > 0 {
|
||||
parentColumn := telemetrylogs.LogsV2BodyV2Column
|
||||
parentColumn := logstelemetryschema.LogsV2BodyV2Column
|
||||
// if the path is already promoted or is being promoted, add it to the promoted column
|
||||
if _, promoted := existingPromotedPaths[it.Path]; promoted || it.Promote {
|
||||
parentColumn = telemetrylogs.LogsV2BodyPromotedColumn
|
||||
parentColumn = logstelemetryschema.LogsV2BodyPromotedColumn
|
||||
}
|
||||
|
||||
for _, index := range it.Indexes {
|
||||
|
||||
@@ -11,8 +11,8 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/modules/services"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -39,7 +39,7 @@ func (m *module) FetchTopLevelOperations(ctx context.Context, start time.Time, s
|
||||
ctx = m.withServicesContext(ctx, "FetchTopLevelOperations")
|
||||
|
||||
db := m.TelemetryStore.ClickhouseDB()
|
||||
query := fmt.Sprintf("SELECT name, serviceName, max(time) as ts FROM %s.%s WHERE time >= @start", telemetrytraces.DBName, telemetrytraces.TopLevelOperationsTableName)
|
||||
query := fmt.Sprintf("SELECT name, serviceName, max(time) as ts FROM %s.%s WHERE time >= @start", tracestelemetryschema.DBName, tracestelemetryschema.TopLevelOperationsTableName)
|
||||
args := []any{clickhouse.Named("start", start)}
|
||||
if len(services) > 0 {
|
||||
query += " AND serviceName IN @services"
|
||||
|
||||
@@ -22,6 +22,13 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/variables"
|
||||
)
|
||||
|
||||
type Handler interface {
|
||||
QueryRange(rw http.ResponseWriter, req *http.Request)
|
||||
QueryRangePreview(rw http.ResponseWriter, req *http.Request)
|
||||
QueryRawStream(rw http.ResponseWriter, req *http.Request)
|
||||
ReplaceVariables(rw http.ResponseWriter, req *http.Request)
|
||||
}
|
||||
|
||||
type handler struct {
|
||||
set factory.ProviderSettings
|
||||
analytics analytics.Analytics
|
||||
@@ -242,7 +249,7 @@ func (handler *handler) ReplaceVariables(rw http.ResponseWriter, req *http.Reque
|
||||
errs := []error{}
|
||||
|
||||
for idx, item := range queryRangeRequest.CompositeQuery.Queries {
|
||||
if item.Type == qbtypes.QueryTypeBuilder {
|
||||
if item.Type == qbtypes.QueryTypeBuilder || item.Type == qbtypes.QueryTypeBuilderAI {
|
||||
switch spec := item.Spec.(type) {
|
||||
case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]:
|
||||
if spec.Filter != nil && spec.Filter.Expression != "" {
|
||||
|
||||
@@ -15,6 +15,14 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// BucketCache is the interface for bucket-based caching.
|
||||
type BucketCache interface {
|
||||
// cached portion + list of gaps to fetch
|
||||
GetMissRanges(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step) (cached *qbtypes.Result, missing []*qbtypes.TimeRange)
|
||||
// store fresh buckets for future hits
|
||||
Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result)
|
||||
}
|
||||
|
||||
// bucketCache implements the BucketCache interface.
|
||||
type bucketCache struct {
|
||||
cache cache.Cache
|
||||
|
||||
@@ -11,8 +11,8 @@ import (
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -277,12 +277,12 @@ func (q *builderQuery[T]) narrowWindowByTraceID(ctx context.Context, fromMS, toM
|
||||
return fromMS, toMS, true, ""
|
||||
}
|
||||
|
||||
traceIDs, found := telemetrytraces.ExtractTraceIDsFromFilter(q.spec.Filter.Expression)
|
||||
traceIDs, found := tracestelemetryschema.ExtractTraceIDsFromFilter(q.spec.Filter.Expression)
|
||||
if !found || len(traceIDs) == 0 {
|
||||
return fromMS, toMS, true, ""
|
||||
}
|
||||
|
||||
finder := telemetrytraces.NewTraceTimeRangeFinder(q.telemetryStore)
|
||||
finder := tracestelemetryschema.NewTraceTimeRangeFinder(q.telemetryStore)
|
||||
traceStart, traceEnd, exists, err := finder.GetTraceTimeRangeMulti(ctx, traceIDs)
|
||||
if err != nil {
|
||||
return fromMS, toMS, true, ""
|
||||
|
||||
@@ -6,18 +6,18 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
func (q *querier) Collect(ctx context.Context, _ valuer.UUID) (map[string]any, error) {
|
||||
stats := make(map[string]any)
|
||||
|
||||
tracesTable := fmt.Sprintf("%s.%s", telemetrytraces.DBName, telemetrytraces.SpanIndexV3TableName)
|
||||
logsTable := fmt.Sprintf("%s.%s", telemetrylogs.DBName, telemetrylogs.LogsV2TableName)
|
||||
metricsTable := fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4TableName)
|
||||
tracesTable := fmt.Sprintf("%s.%s", tracestelemetryschema.DBName, tracestelemetryschema.SpanIndexV3TableName)
|
||||
logsTable := fmt.Sprintf("%s.%s", logstelemetryschema.DBName, logstelemetryschema.LogsV2TableName)
|
||||
metricsTable := fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4TableName)
|
||||
|
||||
var (
|
||||
traces uint64
|
||||
|
||||
@@ -9,12 +9,6 @@ import (
|
||||
|
||||
const DefaultMaxConcurrentQueries = 8
|
||||
|
||||
type SkipResourceFingerprint struct {
|
||||
Enabled bool `yaml:"enabled" mapstructure:"enabled"`
|
||||
// If count of fingerprint is above threshold, skip the fingerprint subquery and filter on main table instead.
|
||||
Threshold uint64 `yaml:"threshold" mapstructure:"threshold"`
|
||||
}
|
||||
|
||||
// Config represents the configuration for the querier.
|
||||
type Config struct {
|
||||
// CacheTTL is the TTL for cached query results
|
||||
@@ -23,8 +17,6 @@ type Config struct {
|
||||
FluxInterval time.Duration `yaml:"flux_interval" mapstructure:"flux_interval"`
|
||||
// MaxConcurrentQueries is the maximum number of concurrent queries for missing ranges
|
||||
MaxConcurrentQueries int `yaml:"max_concurrent_queries" mapstructure:"max_concurrent_queries"`
|
||||
// SkipResourceFingerprint configures when the resource fingerprint subquery is skipped in favor of main-table filtering.
|
||||
SkipResourceFingerprint SkipResourceFingerprint `yaml:"skip_resource_fingerprint" mapstructure:"skip_resource_fingerprint"`
|
||||
// LogTraceIDWindowPadding is the padding added to narrowed down timerange from trace summary to logs with trace_id filter.
|
||||
LogTraceIDWindowPadding time.Duration `yaml:"log_trace_id_window_padding" mapstructure:"log_trace_id_window_padding"`
|
||||
}
|
||||
@@ -39,11 +31,7 @@ func newConfig() factory.Config {
|
||||
// Default values
|
||||
CacheTTL: 168 * time.Hour,
|
||||
FluxInterval: 5 * time.Minute,
|
||||
MaxConcurrentQueries: DefaultMaxConcurrentQueries,
|
||||
SkipResourceFingerprint: SkipResourceFingerprint{
|
||||
Enabled: false,
|
||||
Threshold: 100000,
|
||||
},
|
||||
MaxConcurrentQueries: DefaultMaxConcurrentQueries,
|
||||
LogTraceIDWindowPadding: 5 * time.Minute,
|
||||
}
|
||||
}
|
||||
@@ -59,9 +47,6 @@ func (c Config) Validate() error {
|
||||
if c.MaxConcurrentQueries <= 0 {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "max_concurrent_queries must be positive, got %v", c.MaxConcurrentQueries)
|
||||
}
|
||||
if c.SkipResourceFingerprint.Enabled && c.SkipResourceFingerprint.Threshold == 0 {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "skip_resource_fingerprint.threshold must be > 0 when enabled")
|
||||
}
|
||||
if c.LogTraceIDWindowPadding < 0 {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "log_trace_id_window_padding must not be negative, got %v", c.LogTraceIDWindowPadding)
|
||||
}
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
package querier
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/statsreporter"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// Querier interface defines the contract for querying data.
|
||||
type Querier interface {
|
||||
QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (*qbtypes.QueryRangeResponse, error)
|
||||
QueryRawStream(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, client *qbtypes.RawStream)
|
||||
statsreporter.StatsCollector
|
||||
// QueryRangePreview validates and renders the queries without executing them.
|
||||
QueryRangePreview(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, opts qbtypes.QueryRangePreviewOptions) (*qbtypes.QueryRangePreviewResponse, error)
|
||||
}
|
||||
|
||||
// BucketCache is the interface for bucket-based caching.
|
||||
type BucketCache interface {
|
||||
// cached portion + list of gaps to fetch
|
||||
GetMissRanges(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step) (cached *qbtypes.Result, missing []*qbtypes.TimeRange)
|
||||
// store fresh buckets for future hits
|
||||
Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result)
|
||||
}
|
||||
|
||||
type Handler interface {
|
||||
QueryRange(rw http.ResponseWriter, req *http.Request)
|
||||
// QueryRangePreview is the dry-run endpoint: validate and render without executing.
|
||||
QueryRangePreview(rw http.ResponseWriter, req *http.Request)
|
||||
QueryRawStream(rw http.ResponseWriter, req *http.Request)
|
||||
ReplaceVariables(rw http.ResponseWriter, req *http.Request)
|
||||
}
|
||||
@@ -249,6 +249,7 @@ func (q *querier) buildPreviewProviders(
|
||||
func rendersStandaloneStatement(t qbtypes.QueryType) bool {
|
||||
switch t {
|
||||
case qbtypes.QueryTypeBuilder,
|
||||
qbtypes.QueryTypeBuilderAI,
|
||||
qbtypes.QueryTypePromQL,
|
||||
qbtypes.QueryTypeClickHouseSQL,
|
||||
qbtypes.QueryTypeTraceOperator:
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statsreporter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
@@ -35,6 +36,15 @@ var (
|
||||
intervalWarn = "Query %s is requesting aggregation interval %v seconds, which is smaller than the minimum allowed interval of %v seconds for selected time range. Using the minimum instead"
|
||||
)
|
||||
|
||||
// Querier interface defines the contract for querying data.
|
||||
type Querier interface {
|
||||
QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (*qbtypes.QueryRangeResponse, error)
|
||||
QueryRawStream(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, client *qbtypes.RawStream)
|
||||
statsreporter.StatsCollector
|
||||
// QueryRangePreview validates and renders the queries without executing them.
|
||||
QueryRangePreview(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, opts qbtypes.QueryRangePreviewOptions) (*qbtypes.QueryRangePreviewResponse, error)
|
||||
}
|
||||
|
||||
type querier struct {
|
||||
logger *slog.Logger
|
||||
fl flagger.Flagger
|
||||
@@ -42,6 +52,7 @@ type querier struct {
|
||||
metadataStore telemetrytypes.MetadataStore
|
||||
promEngine prometheus.Prometheus
|
||||
traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
|
||||
aiTraceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
|
||||
logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation]
|
||||
auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation]
|
||||
metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation]
|
||||
@@ -61,6 +72,7 @@ func New(
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
promEngine prometheus.Prometheus,
|
||||
traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
|
||||
aiTraceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
|
||||
logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
|
||||
auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
|
||||
metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation],
|
||||
@@ -82,6 +94,7 @@ func New(
|
||||
metadataStore: metadataStore,
|
||||
promEngine: promEngine,
|
||||
traceStmtBuilder: traceStmtBuilder,
|
||||
aiTraceStmtBuilder: aiTraceStmtBuilder,
|
||||
logStmtBuilder: logStmtBuilder,
|
||||
auditStmtBuilder: auditStmtBuilder,
|
||||
metricStmtBuilder: metricStmtBuilder,
|
||||
@@ -232,6 +245,16 @@ func (q *querier) buildQueries(
|
||||
}
|
||||
queries[traceOpQuery.Name] = toq
|
||||
steps[traceOpQuery.Name] = traceOpQuery.StepInterval
|
||||
case qbtypes.QueryTypeBuilderAI:
|
||||
spec, ok := query.Spec.(qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation])
|
||||
if !ok {
|
||||
return nil, nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid AI builder query spec %T", query.Spec)
|
||||
}
|
||||
spec.ShiftBy = extractShiftFromBuilderQuery(spec)
|
||||
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
|
||||
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.aiTraceStmtBuilder, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
|
||||
queries[spec.Name] = bq
|
||||
steps[spec.Name] = spec.StepInterval
|
||||
case qbtypes.QueryTypeBuilder:
|
||||
switch spec := query.Spec.(type) {
|
||||
case qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]:
|
||||
@@ -298,6 +321,11 @@ func (q *querier) populateQBEvent(event *qbtypes.QBEvent, queries []qbtypes.Quer
|
||||
case qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]:
|
||||
event.MetricsUsed = true
|
||||
}
|
||||
case qbtypes.QueryTypeBuilderAI:
|
||||
filter := query.GetFilter()
|
||||
event.FilterApplied = event.FilterApplied || (filter != nil && filter.Expression != "")
|
||||
event.GroupByApplied = event.GroupByApplied || len(query.GetGroupBy()) > 0
|
||||
event.TracesUsed = true
|
||||
case qbtypes.QueryTypePromQL:
|
||||
event.MetricsUsed = true
|
||||
case qbtypes.QueryTypeTraceOperator:
|
||||
@@ -860,7 +888,9 @@ func (q *querier) createRangedQuery(_ valuer.UUID, originalQuery qbtypes.Query,
|
||||
specCopy := qt.spec.Copy()
|
||||
specCopy.ShiftBy = extractShiftFromBuilderQuery(specCopy)
|
||||
adjustedTimeRange := adjustTimeRangeForShift(specCopy, timeRange, qt.kind)
|
||||
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, q.traceStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
|
||||
// Reuse the statement builder the original query was created with, so an AI
|
||||
// query keeps its AI builder without re-deriving it from the spec.
|
||||
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, qt.stmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
|
||||
|
||||
case *builderQuery[qbtypes.LogAggregation]:
|
||||
specCopy := qt.spec.Copy()
|
||||
@@ -1217,6 +1247,8 @@ func (q *querier) adjustStepInterval(queries []qbtypes.QueryEnvelope, start, end
|
||||
if qe.GetStepInterval().Seconds() == 0 {
|
||||
qe.SetStepInterval(secondsStep(metricRecommended))
|
||||
}
|
||||
case qbtypes.QueryTypeBuilderAI:
|
||||
clampStep(qe, traceLogRecommended, traceLogMin, &warnings)
|
||||
case qbtypes.QueryTypeTraceOperator:
|
||||
clampStep(qe, traceLogRecommended, traceLogMin, &warnings)
|
||||
}
|
||||
|
||||
@@ -49,6 +49,7 @@ func TestQueryRange_MetricTypeMissing(t *testing.T) {
|
||||
metadataStore,
|
||||
nil, // prometheus
|
||||
nil, // traceStmtBuilder
|
||||
nil, // aiTraceStmtBuilder
|
||||
nil, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
nil, // metricStmtBuilder
|
||||
@@ -119,17 +120,18 @@ func TestQueryRange_MetricTypeFromStore(t *testing.T) {
|
||||
providerSettings,
|
||||
telemetryStore,
|
||||
metadataStore,
|
||||
nil, // prometheus
|
||||
nil, // traceStmtBuilder
|
||||
nil, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
&mockMetricStmtBuilder{}, // metricStmtBuilder
|
||||
nil, // meterStmtBuilder
|
||||
nil, // traceOperatorStmtBuilder
|
||||
nil, // bucketCache
|
||||
flaggertest.New(t), // flagger
|
||||
0, // logTraceIDWindowPadding
|
||||
0, // maxConcurrentQueries
|
||||
nil, // prometheus
|
||||
nil, // traceStmtBuilder
|
||||
nil, // aiTraceStmtBuilder
|
||||
nil, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
&mockMetricStmtBuilder{},
|
||||
nil, // meterStmtBuilder
|
||||
nil, // traceOperatorStmtBuilder
|
||||
nil, // bucketCache
|
||||
flaggertest.New(t), // flagger
|
||||
0, // logTraceIDWindowPadding
|
||||
0, // maxConcurrentQueries
|
||||
)
|
||||
|
||||
req := &qbtypes.QueryRangeRequest{
|
||||
|
||||
52
pkg/querier/queriertest/querier.go
Normal file
52
pkg/querier/queriertest/querier.go
Normal file
@@ -0,0 +1,52 @@
|
||||
package queriertest
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// MockQuerier implements querier.Querier for testing. Each behavior can be
|
||||
// overridden via a function field; unset fields return empty, non-error values.
|
||||
type MockQuerier struct {
|
||||
QueryRangeFunc func(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (*qbtypes.QueryRangeResponse, error)
|
||||
QueryRawStreamFunc func(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, client *qbtypes.RawStream)
|
||||
QueryRangePreviewFunc func(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, opts qbtypes.QueryRangePreviewOptions) (*qbtypes.QueryRangePreviewResponse, error)
|
||||
CollectFunc func(ctx context.Context, orgID valuer.UUID) (map[string]any, error)
|
||||
}
|
||||
|
||||
var _ querier.Querier = (*MockQuerier)(nil)
|
||||
|
||||
// NewMockQuerier creates a new MockQuerier.
|
||||
func NewMockQuerier() *MockQuerier {
|
||||
return &MockQuerier{}
|
||||
}
|
||||
|
||||
func (m *MockQuerier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (*qbtypes.QueryRangeResponse, error) {
|
||||
if m.QueryRangeFunc != nil {
|
||||
return m.QueryRangeFunc(ctx, orgID, req)
|
||||
}
|
||||
return &qbtypes.QueryRangeResponse{}, nil
|
||||
}
|
||||
|
||||
func (m *MockQuerier) QueryRawStream(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, client *qbtypes.RawStream) {
|
||||
if m.QueryRawStreamFunc != nil {
|
||||
m.QueryRawStreamFunc(ctx, orgID, req, client)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *MockQuerier) QueryRangePreview(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, opts qbtypes.QueryRangePreviewOptions) (*qbtypes.QueryRangePreviewResponse, error) {
|
||||
if m.QueryRangePreviewFunc != nil {
|
||||
return m.QueryRangePreviewFunc(ctx, orgID, req, opts)
|
||||
}
|
||||
return &qbtypes.QueryRangePreviewResponse{}, nil
|
||||
}
|
||||
|
||||
func (m *MockQuerier) Collect(ctx context.Context, orgID valuer.UUID) (map[string]any, error) {
|
||||
if m.CollectFunc != nil {
|
||||
return m.CollectFunc(ctx, orgID)
|
||||
}
|
||||
return map[string]any{}, nil
|
||||
}
|
||||
@@ -3,192 +3,56 @@ package signozquerier
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/cache"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryaudit"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetadata"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
// NewFactory creates a new factory for the signoz querier provider.
|
||||
// NewFactory creates a new factory for the signoz querier provider. The query
|
||||
// stack (metadata store, statement builders, bucket cache) is assembled once in
|
||||
// signoz.go and injected here.
|
||||
func NewFactory(
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
prometheus prometheus.Prometheus,
|
||||
cache cache.Cache,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
|
||||
aiTraceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
|
||||
logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
|
||||
auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
|
||||
metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation],
|
||||
meterStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation],
|
||||
traceOperatorStmtBuilder qbtypes.TraceOperatorStatementBuilder,
|
||||
bucketCache querier.BucketCache,
|
||||
flagger flagger.Flagger,
|
||||
) factory.ProviderFactory[querier.Querier, querier.Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("signoz"),
|
||||
func(
|
||||
ctx context.Context,
|
||||
_ context.Context,
|
||||
settings factory.ProviderSettings,
|
||||
cfg querier.Config,
|
||||
) (querier.Querier, error) {
|
||||
return newProvider(ctx, settings, cfg, telemetryStore, prometheus, cache, flagger)
|
||||
return querier.New(
|
||||
settings,
|
||||
telemetryStore,
|
||||
metadataStore,
|
||||
prometheus,
|
||||
traceStmtBuilder,
|
||||
aiTraceStmtBuilder,
|
||||
logStmtBuilder,
|
||||
auditStmtBuilder,
|
||||
metricStmtBuilder,
|
||||
meterStmtBuilder,
|
||||
traceOperatorStmtBuilder,
|
||||
bucketCache,
|
||||
flagger,
|
||||
cfg.LogTraceIDWindowPadding,
|
||||
cfg.MaxConcurrentQueries,
|
||||
), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func newProvider(
|
||||
_ context.Context,
|
||||
settings factory.ProviderSettings,
|
||||
cfg querier.Config,
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
prometheus prometheus.Prometheus,
|
||||
cache cache.Cache,
|
||||
flagger flagger.Flagger,
|
||||
) (querier.Querier, error) {
|
||||
|
||||
// Create telemetry metadata store
|
||||
telemetryMetadataStore := telemetrymetadata.NewTelemetryMetaStore(
|
||||
settings,
|
||||
telemetryStore,
|
||||
telemetrytraces.DBName,
|
||||
telemetrytraces.TagAttributesV2TableName,
|
||||
telemetrytraces.SpanAttributesKeysTblName,
|
||||
telemetrytraces.SpanIndexV3TableName,
|
||||
telemetrymetrics.DBName,
|
||||
telemetrymetrics.AttributesMetadataTableName,
|
||||
telemetrymeter.DBName,
|
||||
telemetrymeter.SamplesAgg1dTableName,
|
||||
telemetrylogs.DBName,
|
||||
telemetrylogs.LogsV2TableName,
|
||||
telemetrylogs.TagAttributesV2TableName,
|
||||
telemetrylogs.LogAttributeKeysTblName,
|
||||
telemetrylogs.LogResourceKeysTblName,
|
||||
telemetryaudit.DBName,
|
||||
telemetryaudit.AuditLogsTableName,
|
||||
telemetryaudit.TagAttributesTableName,
|
||||
telemetryaudit.LogAttributeKeysTblName,
|
||||
telemetryaudit.LogResourceKeysTblName,
|
||||
telemetrymetadata.DBName,
|
||||
telemetrymetadata.AttributesMetadataTableName,
|
||||
telemetrymetadata.ColumnEvolutionMetadataTableName,
|
||||
flagger,
|
||||
)
|
||||
|
||||
// Create trace statement builder
|
||||
traceFieldMapper := telemetrytraces.NewFieldMapper()
|
||||
traceConditionBuilder := telemetrytraces.NewConditionBuilder(traceFieldMapper)
|
||||
|
||||
traceAggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, traceFieldMapper, traceConditionBuilder, flagger)
|
||||
traceStmtBuilder := telemetrytraces.NewTraceQueryStatementBuilder(
|
||||
settings,
|
||||
telemetryMetadataStore,
|
||||
traceFieldMapper,
|
||||
traceConditionBuilder,
|
||||
traceAggExprRewriter,
|
||||
telemetryStore,
|
||||
flagger,
|
||||
cfg.SkipResourceFingerprint.Enabled,
|
||||
cfg.SkipResourceFingerprint.Threshold,
|
||||
)
|
||||
|
||||
// Create trace operator statement builder
|
||||
traceOperatorStmtBuilder := telemetrytraces.NewTraceOperatorStatementBuilder(
|
||||
settings,
|
||||
telemetryMetadataStore,
|
||||
traceFieldMapper,
|
||||
traceConditionBuilder,
|
||||
traceStmtBuilder,
|
||||
traceAggExprRewriter,
|
||||
flagger,
|
||||
)
|
||||
|
||||
// Create log statement builder
|
||||
logFieldMapper := telemetrylogs.NewFieldMapper(flagger)
|
||||
logConditionBuilder := telemetrylogs.NewConditionBuilder(logFieldMapper, flagger)
|
||||
logAggExprRewriter := querybuilder.NewAggExprRewriter(
|
||||
settings,
|
||||
telemetrylogs.DefaultFullTextColumn,
|
||||
logFieldMapper,
|
||||
logConditionBuilder,
|
||||
flagger,
|
||||
)
|
||||
logStmtBuilder := telemetrylogs.NewLogQueryStatementBuilder(
|
||||
settings,
|
||||
telemetryMetadataStore,
|
||||
logFieldMapper,
|
||||
logConditionBuilder,
|
||||
logAggExprRewriter,
|
||||
telemetrylogs.DefaultFullTextColumn,
|
||||
flagger,
|
||||
telemetryStore,
|
||||
cfg.SkipResourceFingerprint.Enabled,
|
||||
cfg.SkipResourceFingerprint.Threshold,
|
||||
)
|
||||
|
||||
// Create audit statement builder
|
||||
auditFieldMapper := telemetryaudit.NewFieldMapper()
|
||||
auditConditionBuilder := telemetryaudit.NewConditionBuilder(auditFieldMapper)
|
||||
auditAggExprRewriter := querybuilder.NewAggExprRewriter(
|
||||
settings,
|
||||
telemetryaudit.DefaultFullTextColumn,
|
||||
auditFieldMapper,
|
||||
auditConditionBuilder,
|
||||
flagger,
|
||||
)
|
||||
auditStmtBuilder := telemetryaudit.NewAuditQueryStatementBuilder(
|
||||
settings,
|
||||
telemetryMetadataStore,
|
||||
auditFieldMapper,
|
||||
auditConditionBuilder,
|
||||
auditAggExprRewriter,
|
||||
telemetryaudit.DefaultFullTextColumn,
|
||||
flagger,
|
||||
)
|
||||
|
||||
// Create metric statement builder
|
||||
metricFieldMapper := telemetrymetrics.NewFieldMapper()
|
||||
metricConditionBuilder := telemetrymetrics.NewConditionBuilder(metricFieldMapper)
|
||||
metricStmtBuilder := telemetrymetrics.NewMetricQueryStatementBuilder(
|
||||
settings,
|
||||
telemetryMetadataStore,
|
||||
metricFieldMapper,
|
||||
metricConditionBuilder,
|
||||
flagger,
|
||||
)
|
||||
|
||||
// Create meter statement builder
|
||||
meterStmtBuilder := telemetrymeter.NewMeterQueryStatementBuilder(
|
||||
settings,
|
||||
telemetryMetadataStore,
|
||||
metricFieldMapper,
|
||||
metricConditionBuilder,
|
||||
metricStmtBuilder,
|
||||
)
|
||||
|
||||
// Create bucket cache
|
||||
bucketCache := querier.NewBucketCache(
|
||||
settings,
|
||||
cache,
|
||||
cfg.CacheTTL,
|
||||
cfg.FluxInterval,
|
||||
)
|
||||
|
||||
// Create and return the querier
|
||||
return querier.New(
|
||||
settings,
|
||||
telemetryStore,
|
||||
telemetryMetadataStore,
|
||||
prometheus,
|
||||
traceStmtBuilder,
|
||||
logStmtBuilder,
|
||||
auditStmtBuilder,
|
||||
metricStmtBuilder,
|
||||
meterStmtBuilder,
|
||||
traceOperatorStmtBuilder,
|
||||
bucketCache,
|
||||
flagger,
|
||||
cfg.LogTraceIDWindowPadding,
|
||||
cfg.MaxConcurrentQueries,
|
||||
), nil
|
||||
}
|
||||
|
||||
@@ -18,6 +18,14 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/querier/signozquerier"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore/sqlstoretest"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/aistatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/auditstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/logsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/meterstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/tracesstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetadata"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -105,7 +113,25 @@ func NewTestManager(t *testing.T, testOpts *TestManagerOptions) *Manager {
|
||||
}
|
||||
|
||||
// Create querier with test values
|
||||
providerFactory := signozquerier.NewFactory(telemetryStore, prometheus, cache, flagger)
|
||||
metadataStore := telemetrymetadata.NewTelemetryMetaStore(providerSettings, telemetryStore, flagger)
|
||||
cfg := statementbuilder.Config{}
|
||||
ctx := context.Background()
|
||||
traceStmtBuilder, err := tracesstatementbuilder.NewFactory(telemetryStore, metadataStore, flagger).New(ctx, providerSettings, cfg)
|
||||
require.NoError(t, err)
|
||||
aiTraceStmtBuilder, err := aistatementbuilder.NewFactory(telemetryStore, metadataStore, flagger).New(ctx, providerSettings, cfg)
|
||||
require.NoError(t, err)
|
||||
traceOperatorStmtBuilder, err := tracesstatementbuilder.NewOperatorFactory(telemetryStore, metadataStore, flagger).New(ctx, providerSettings, cfg)
|
||||
require.NoError(t, err)
|
||||
logStmtBuilder, err := logsstatementbuilder.NewFactory(telemetryStore, metadataStore, flagger).New(ctx, providerSettings, cfg)
|
||||
require.NoError(t, err)
|
||||
auditStmtBuilder, err := auditstatementbuilder.NewFactory(metadataStore, flagger).New(ctx, providerSettings, cfg)
|
||||
require.NoError(t, err)
|
||||
metricStmtBuilder, err := metricsstatementbuilder.NewFactory(metadataStore, flagger).New(ctx, providerSettings, cfg)
|
||||
require.NoError(t, err)
|
||||
meterStmtBuilder, err := meterstatementbuilder.NewFactory(metadataStore, flagger).New(ctx, providerSettings, cfg)
|
||||
require.NoError(t, err)
|
||||
bucketCache := querier.NewBucketCache(providerSettings, cache, 0, 0)
|
||||
providerFactory := signozquerier.NewFactory(telemetryStore, prometheus, metadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger)
|
||||
mockQuerier, err := providerFactory.New(context.Background(), providerSettings, querier.Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
|
||||
@@ -8,11 +8,11 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/logsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/tracesstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -24,7 +24,7 @@ func prepareQuerierForMetrics(t *testing.T, telemetryStore telemetrystore.Teleme
|
||||
providerSettings := instrumentationtest.New().ToProviderSettings()
|
||||
metadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
|
||||
flagger, err := flagger.New(
|
||||
fl, err := flagger.New(
|
||||
context.Background(),
|
||||
instrumentationtest.New().ToProviderSettings(),
|
||||
flagger.Config{},
|
||||
@@ -32,15 +32,8 @@ func prepareQuerierForMetrics(t *testing.T, telemetryStore telemetrystore.Teleme
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
metricFieldMapper := telemetrymetrics.NewFieldMapper()
|
||||
metricConditionBuilder := telemetrymetrics.NewConditionBuilder(metricFieldMapper)
|
||||
metricStmtBuilder := telemetrymetrics.NewMetricQueryStatementBuilder(
|
||||
providerSettings,
|
||||
metadataStore,
|
||||
metricFieldMapper,
|
||||
metricConditionBuilder,
|
||||
flagger,
|
||||
)
|
||||
metricStmtBuilder, err := metricsstatementbuilder.NewFactory(metadataStore, fl).New(context.Background(), providerSettings, statementbuilder.Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
return querier.New(
|
||||
providerSettings,
|
||||
@@ -48,13 +41,14 @@ func prepareQuerierForMetrics(t *testing.T, telemetryStore telemetrystore.Teleme
|
||||
metadataStore,
|
||||
nil, // prometheus
|
||||
nil, // traceStmtBuilder
|
||||
nil, // aiTraceStmtBuilder
|
||||
nil, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
metricStmtBuilder,
|
||||
nil, // meterStmtBuilder
|
||||
nil, // traceOperatorStmtBuilder
|
||||
nil, // bucketCache
|
||||
flagger,
|
||||
fl,
|
||||
0,
|
||||
0, // maxConcurrentQueries (0 means default)
|
||||
), metadataStore
|
||||
@@ -73,40 +67,22 @@ func prepareQuerierForLogs(t *testing.T, telemetryStore telemetrystore.Telemetry
|
||||
metadataStore.KeysMap = keysMap
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
logFieldMapper := telemetrylogs.NewFieldMapper(fl)
|
||||
logConditionBuilder := telemetrylogs.NewConditionBuilder(logFieldMapper, fl)
|
||||
logAggExprRewriter := querybuilder.NewAggExprRewriter(
|
||||
providerSettings,
|
||||
telemetrylogs.DefaultFullTextColumn,
|
||||
logFieldMapper,
|
||||
logConditionBuilder,
|
||||
fl,
|
||||
)
|
||||
logStmtBuilder := telemetrylogs.NewLogQueryStatementBuilder(
|
||||
providerSettings,
|
||||
metadataStore,
|
||||
logFieldMapper,
|
||||
logConditionBuilder,
|
||||
logAggExprRewriter,
|
||||
telemetrylogs.DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
100000,
|
||||
)
|
||||
logStmtBuilder, err := logsstatementbuilder.NewFactory(nil, metadataStore, fl).New(context.Background(), providerSettings, statementbuilder.Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
return querier.New(
|
||||
providerSettings,
|
||||
telemetryStore,
|
||||
metadataStore,
|
||||
nil, // prometheus
|
||||
nil, // traceStmtBuilder
|
||||
logStmtBuilder, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
nil, // metricStmtBuilder
|
||||
nil, // meterStmtBuilder
|
||||
nil, // traceOperatorStmtBuilder
|
||||
nil, // bucketCache
|
||||
nil, // prometheus
|
||||
nil, // traceStmtBuilder
|
||||
nil, // aiTraceStmtBuilder
|
||||
logStmtBuilder,
|
||||
nil, // auditStmtBuilder
|
||||
nil, // metricStmtBuilder
|
||||
nil, // meterStmtBuilder
|
||||
nil, // traceOperatorStmtBuilder
|
||||
nil, // bucketCache
|
||||
fl,
|
||||
5*time.Minute, // logTraceIDWindowPadding
|
||||
0, // maxConcurrentQueries (0 means default)
|
||||
@@ -126,36 +102,23 @@ func prepareQuerierForTraces(t *testing.T, telemetryStore telemetrystore.Telemet
|
||||
}
|
||||
metadataStore.KeysMap = keysMap
|
||||
|
||||
// Create trace statement builder
|
||||
traceFieldMapper := telemetrytraces.NewFieldMapper()
|
||||
traceConditionBuilder := telemetrytraces.NewConditionBuilder(traceFieldMapper)
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
traceAggExprRewriter := querybuilder.NewAggExprRewriter(providerSettings, nil, traceFieldMapper, traceConditionBuilder, fl)
|
||||
traceStmtBuilder := telemetrytraces.NewTraceQueryStatementBuilder(
|
||||
providerSettings,
|
||||
metadataStore,
|
||||
traceFieldMapper,
|
||||
traceConditionBuilder,
|
||||
traceAggExprRewriter,
|
||||
telemetryStore,
|
||||
fl,
|
||||
false,
|
||||
100000,
|
||||
)
|
||||
traceStmtBuilder, err := tracesstatementbuilder.NewFactory(telemetryStore, metadataStore, fl).New(context.Background(), providerSettings, statementbuilder.Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
return querier.New(
|
||||
providerSettings,
|
||||
telemetryStore,
|
||||
metadataStore,
|
||||
nil, // prometheus
|
||||
traceStmtBuilder, // traceStmtBuilder
|
||||
nil, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
nil, // metricStmtBuilder
|
||||
nil, // meterStmtBuilder
|
||||
nil, // traceOperatorStmtBuilder
|
||||
nil, // bucketCache
|
||||
nil, // prometheus
|
||||
traceStmtBuilder,
|
||||
nil, // aiTraceStmtBuilder
|
||||
nil, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
nil, // metricStmtBuilder
|
||||
nil, // meterStmtBuilder
|
||||
nil, // traceOperatorStmtBuilder
|
||||
nil, // bucketCache
|
||||
fl,
|
||||
0,
|
||||
0, // maxConcurrentQueries (0 means default)
|
||||
|
||||
176
pkg/querybuilder/filter_split.go
Normal file
176
pkg/querybuilder/filter_split.go
Normal file
@@ -0,0 +1,176 @@
|
||||
package querybuilder
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/antlr4-go/antlr/v4"
|
||||
)
|
||||
|
||||
// SplitFilterForAggregates partitions a single filter expression into a span-level
|
||||
// part (a WHERE over spans) and a trace-level part (a HAVING over per-trace
|
||||
// aggregates), splitting on the top-level AND.
|
||||
//
|
||||
// A key is trace-level when it carries the trace field context (`trace.completion_tokens`)
|
||||
// or, with no context, its bare name is in aggregateNames. Any other explicit context
|
||||
// (`span.`, `resource.`, …) is span-level. Trace-level and span-level keys may be
|
||||
// AND-combined (they run at different query stages) but not OR-combined; an OR that
|
||||
// mixes the two is an error.
|
||||
func SplitFilterForAggregates(query string, aggregateNames map[string]struct{}) (spanExpr string, havingExpr string, err error) {
|
||||
if strings.TrimSpace(query) == "" {
|
||||
return "", "", nil
|
||||
}
|
||||
|
||||
tree, syntaxErrors := parseFilterQuery(query)
|
||||
if len(syntaxErrors) > 0 {
|
||||
combinedErrors := errors.Newf(
|
||||
errors.TypeInvalidInput,
|
||||
errors.CodeInvalidInput,
|
||||
"Found %d syntax errors while parsing the filter expression.",
|
||||
len(syntaxErrors),
|
||||
)
|
||||
additionals := make([]string, 0, len(syntaxErrors))
|
||||
for _, syntaxError := range syntaxErrors {
|
||||
if syntaxError.Error() != "" {
|
||||
additionals = append(additionals, syntaxError.Error())
|
||||
}
|
||||
}
|
||||
// TODO: add troubleshooting link to the filter query syntax guide once it's published.
|
||||
return "", "", combinedErrors.WithAdditional(additionals...)
|
||||
}
|
||||
|
||||
s := filterSplitter{query: []rune(query), aggregateNames: aggregateNames}
|
||||
s.visit(tree)
|
||||
|
||||
if s.mixed {
|
||||
return "", "", errors.NewInvalidInputf(errors.CodeInvalidInput,
|
||||
"trace-level and span-level filters cannot be combined within an OR/NOT group; separate them with a top-level AND")
|
||||
}
|
||||
return strings.Join(s.span, " AND "), strings.Join(s.having, " AND "), nil
|
||||
}
|
||||
|
||||
func parseFilterQuery(query string) (antlr.Tree, []*SyntaxErr) {
|
||||
lexerErrorListener := NewErrorListener()
|
||||
lexer := grammar.NewFilterQueryLexer(antlr.NewInputStream(query))
|
||||
lexer.RemoveErrorListeners()
|
||||
lexer.AddErrorListener(lexerErrorListener)
|
||||
|
||||
parserErrorListener := NewErrorListener()
|
||||
parser := grammar.NewFilterQueryParser(antlr.NewCommonTokenStream(lexer, 0))
|
||||
parser.RemoveErrorListeners()
|
||||
parser.AddErrorListener(parserErrorListener)
|
||||
|
||||
tree := parser.Query()
|
||||
return tree, append(lexerErrorListener.SyntaxErrors, parserErrorListener.SyntaxErrors...)
|
||||
}
|
||||
|
||||
// filterSplitter walks the parse tree once, flattening the top-level AND chain and
|
||||
// routing each atom (a comparison, a NOT expression, or a whole multi-branch OR group)
|
||||
// to the span or having bucket by the class of the keys it references.
|
||||
type filterSplitter struct {
|
||||
query []rune
|
||||
aggregateNames map[string]struct{}
|
||||
span []string
|
||||
having []string
|
||||
mixed bool
|
||||
}
|
||||
|
||||
func (s *filterSplitter) visit(node antlr.Tree) {
|
||||
switch n := node.(type) {
|
||||
case *grammar.QueryContext:
|
||||
if n.Expression() != nil {
|
||||
s.visit(n.Expression())
|
||||
}
|
||||
case *grammar.ExpressionContext:
|
||||
if n.OrExpression() != nil {
|
||||
s.visit(n.OrExpression())
|
||||
}
|
||||
case *grammar.OrExpressionContext:
|
||||
// a single branch is just an AND chain; multiple branches are a real OR, kept
|
||||
// whole so a class-mixing OR can be rejected.
|
||||
if ands := n.AllAndExpression(); len(ands) == 1 {
|
||||
s.visit(ands[0])
|
||||
} else {
|
||||
s.route(n)
|
||||
}
|
||||
case *grammar.AndExpressionContext:
|
||||
for _, u := range n.AllUnaryExpression() {
|
||||
s.visit(u)
|
||||
}
|
||||
case *grammar.UnaryExpressionContext:
|
||||
if n.NOT() != nil {
|
||||
s.route(n)
|
||||
} else if n.Primary() != nil {
|
||||
s.visit(n.Primary())
|
||||
}
|
||||
case *grammar.PrimaryContext:
|
||||
if n.OrExpression() != nil { // parenthesized sub-expression
|
||||
s.visit(n.OrExpression())
|
||||
} else {
|
||||
s.route(n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// route classifies an atom and appends its original source text to the right bucket.
|
||||
func (s *filterSplitter) route(atom antlr.ParserRuleContext) {
|
||||
isTrace, isSpan := classifyKeys(atom, s.aggregateNames)
|
||||
if isTrace && isSpan {
|
||||
s.mixed = true
|
||||
return
|
||||
}
|
||||
text := atomSourceText(s.query, atom)
|
||||
// A multi-branch OR group's source slice excludes its enclosing parens (they belong
|
||||
// to the parent Primary). Re-wrap it so rejoining a bucket with " AND " cannot invert
|
||||
// OR/AND precedence, e.g. `a AND (b OR c)` must not flatten to `a AND b OR c`.
|
||||
if or, ok := atom.(*grammar.OrExpressionContext); ok && len(or.AllAndExpression()) > 1 {
|
||||
text = "(" + text + ")"
|
||||
}
|
||||
if isTrace {
|
||||
s.having = append(s.having, text)
|
||||
} else {
|
||||
s.span = append(s.span, text)
|
||||
}
|
||||
}
|
||||
|
||||
// classifyKeys reports whether a subtree references trace-level and/or span-level keys.
|
||||
// A key is trace-level when it carries the trace field context or, with no context,
|
||||
// its name is a known aggregate; an unknown name under the trace context stays
|
||||
// trace-level so the aggregate validation rejects it with a targeted error. Any other
|
||||
// explicit context (`span.`, `resource.`, …) is span-level.
|
||||
func classifyKeys(node antlr.Tree, aggregateNames map[string]struct{}) (isTrace, isSpan bool) {
|
||||
kc, ok := node.(*grammar.KeyContext)
|
||||
if ok {
|
||||
key := telemetrytypes.GetFieldKeyFromKeyText(kc.GetText())
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextTrace:
|
||||
isTrace = true
|
||||
case telemetrytypes.FieldContextUnspecified:
|
||||
_, isTrace = aggregateNames[key.Name]
|
||||
isSpan = !isTrace
|
||||
default:
|
||||
isSpan = true
|
||||
}
|
||||
return
|
||||
}
|
||||
for i := 0; i < node.GetChildCount(); i++ {
|
||||
t, s := classifyKeys(node.GetChild(i), aggregateNames)
|
||||
isTrace = isTrace || t
|
||||
isSpan = isSpan || s
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// atomSourceText returns the original source substring for an atom, preserving
|
||||
// whitespace. The token stream drops skipped whitespace, which would glue word
|
||||
// operators (OR/AND/NOT) to their operands, so slice the input by token offsets.
|
||||
// ANTLR offsets are rune indices (InputStream holds []rune), hence the rune slice.
|
||||
func atomSourceText(query []rune, atom antlr.ParserRuleContext) string {
|
||||
start, stop := atom.GetStart(), atom.GetStop()
|
||||
if start == nil || stop == nil || start.GetStart() < 0 || stop.GetStop() >= len(query) || stop.GetStop() < start.GetStart() {
|
||||
return atom.GetText()
|
||||
}
|
||||
return string(query[start.GetStart() : stop.GetStop()+1])
|
||||
}
|
||||
222
pkg/querybuilder/filter_split_test.go
Normal file
222
pkg/querybuilder/filter_split_test.go
Normal file
@@ -0,0 +1,222 @@
|
||||
package querybuilder
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSplitFilterForAggregates(t *testing.T) {
|
||||
agg := map[string]struct{}{"completion_tokens": {}, "span_count": {}, "prompt_tokens": {}}
|
||||
|
||||
type tc struct {
|
||||
name string
|
||||
query string
|
||||
span string // expected span-level (WHERE) part; "" => empty
|
||||
having string // expected trace-level (HAVING) part; "" => empty
|
||||
wantErr bool
|
||||
}
|
||||
|
||||
cases := []tc{
|
||||
// --- empty input ---------------------------------------------------------
|
||||
{
|
||||
name: "empty",
|
||||
},
|
||||
{
|
||||
name: "whitespace only",
|
||||
query: " ",
|
||||
},
|
||||
|
||||
// --- single class --------------------------------------------------------
|
||||
{
|
||||
name: "span only",
|
||||
query: "service.name = 'x'",
|
||||
span: "service.name = 'x'",
|
||||
},
|
||||
{
|
||||
name: "agg only bare",
|
||||
query: "completion_tokens > 1000",
|
||||
having: "completion_tokens > 1000",
|
||||
},
|
||||
{
|
||||
// the user-facing `trace.` prefix marks a trace-level aggregate.
|
||||
name: "agg only trace prefix",
|
||||
query: "trace.completion_tokens > 1000",
|
||||
having: "trace.completion_tokens > 1000",
|
||||
},
|
||||
{
|
||||
// an unknown name under the trace context still routes trace-level, so the
|
||||
// aggregate validation rejects it with a targeted error instead of the span
|
||||
// path failing on an unknown field.
|
||||
name: "unknown aggregate under trace context stays trace-level",
|
||||
query: "trace.not_an_aggregate > 1000",
|
||||
having: "trace.not_an_aggregate > 1000",
|
||||
},
|
||||
|
||||
{
|
||||
// ANTLR token offsets are rune indices; slicing must not shift after a
|
||||
// multi-byte char (this used to truncate 1000 → 100).
|
||||
name: "unicode value before the split",
|
||||
query: "service.name = 'héllo' AND completion_tokens > 1000",
|
||||
span: "service.name = 'héllo'",
|
||||
having: "completion_tokens > 1000",
|
||||
},
|
||||
|
||||
// --- top-level AND splits across the two buckets -------------------------
|
||||
{
|
||||
name: "span AND agg",
|
||||
query: "service.name = 'x' AND completion_tokens > 1000",
|
||||
span: "service.name = 'x'",
|
||||
having: "completion_tokens > 1000",
|
||||
},
|
||||
{
|
||||
// order within a bucket is preserved; the two span atoms join with AND.
|
||||
name: "span AND span AND agg",
|
||||
query: "service.name = 'x' AND kind_string = 'Internal' AND completion_tokens > 1000",
|
||||
span: "service.name = 'x' AND kind_string = 'Internal'",
|
||||
having: "completion_tokens > 1000",
|
||||
},
|
||||
{
|
||||
// a parenthesized top-level AND still splits across the two buckets.
|
||||
name: "parenthesized span AND agg",
|
||||
query: "(service.name = 'x' AND completion_tokens > 1000)",
|
||||
span: "service.name = 'x'",
|
||||
having: "completion_tokens > 1000",
|
||||
},
|
||||
|
||||
// --- OR groups are re-wrapped in parens so a later AND-join can't invert
|
||||
// precedence (`a AND (b OR c)` must not flatten to `a AND b OR c`) ------
|
||||
{
|
||||
name: "agg OR agg",
|
||||
query: "completion_tokens > 1000 OR span_count > 3",
|
||||
having: "(completion_tokens > 1000 OR span_count > 3)",
|
||||
},
|
||||
{
|
||||
name: "span OR span",
|
||||
query: "service.name = 'x' OR kind_string = 'Internal'",
|
||||
span: "(service.name = 'x' OR kind_string = 'Internal')",
|
||||
},
|
||||
{
|
||||
name: "span AND (span OR span)",
|
||||
query: "service.name = 'x' AND (kind_string = 'Internal' OR kind_string = 'Client')",
|
||||
span: "service.name = 'x' AND (kind_string = 'Internal' OR kind_string = 'Client')",
|
||||
},
|
||||
{
|
||||
name: "agg AND (agg OR agg)",
|
||||
query: "prompt_tokens > 5 AND (completion_tokens > 1000 OR span_count > 3)",
|
||||
having: "prompt_tokens > 5 AND (completion_tokens > 1000 OR span_count > 3)",
|
||||
},
|
||||
{
|
||||
// the OR group routes to span, the trailing aggregate to having.
|
||||
name: "span AND (span OR span) AND agg",
|
||||
query: "a.b = 'x' AND (c.d = 'y' OR e.f = 'z') AND completion_tokens > 1000",
|
||||
span: "a.b = 'x' AND (c.d = 'y' OR e.f = 'z')",
|
||||
having: "completion_tokens > 1000",
|
||||
},
|
||||
|
||||
// --- a nested AND group flattens across the buckets (no spurious parens) --
|
||||
{
|
||||
name: "(span AND agg) AND agg",
|
||||
query: "(service.name = 'x' AND completion_tokens > 1000) AND prompt_tokens > 5",
|
||||
span: "service.name = 'x'",
|
||||
having: "completion_tokens > 1000 AND prompt_tokens > 5",
|
||||
},
|
||||
|
||||
// --- NOT wrapping a single-class group is routed whole to that class ------
|
||||
{
|
||||
name: "not agg",
|
||||
query: "NOT (completion_tokens > 1000)",
|
||||
having: "NOT (completion_tokens > 1000)",
|
||||
},
|
||||
{
|
||||
name: "not span",
|
||||
query: "NOT (service.name = 'x')",
|
||||
span: "NOT (service.name = 'x')",
|
||||
},
|
||||
|
||||
// --- an explicit non-trace context escapes the aggregate-alias shadow -----
|
||||
{
|
||||
// a span attribute named like an aggregate stays reachable via `attribute.`.
|
||||
name: "attribute prefix on aggregate name routes span-level",
|
||||
query: "attribute.completion_tokens > 5",
|
||||
span: "attribute.completion_tokens > 5",
|
||||
},
|
||||
{
|
||||
name: "span prefix on aggregate name routes span-level",
|
||||
query: "span.completion_tokens > 5",
|
||||
span: "span.completion_tokens > 5",
|
||||
},
|
||||
{
|
||||
name: "prefixed attribute AND bare aggregate split across buckets",
|
||||
query: "attribute.completion_tokens > 5 AND completion_tokens > 1000",
|
||||
span: "attribute.completion_tokens > 5",
|
||||
having: "completion_tokens > 1000",
|
||||
},
|
||||
|
||||
// --- class-mixing is rejected in an OR group, a NOT group, or a nested OR -
|
||||
{
|
||||
name: "agg OR span rejected",
|
||||
query: "completion_tokens > 1000 OR service.name = 'x'",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "not mixed rejected",
|
||||
query: "NOT (completion_tokens > 1000 AND service.name = 'x')",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "span AND (agg OR span) rejected",
|
||||
query: "service.name = 'x' AND (completion_tokens > 1000 OR kind_string = 'Client')",
|
||||
wantErr: true,
|
||||
},
|
||||
|
||||
// --- syntax errors are rejected, not silently dropped by error recovery ---
|
||||
{
|
||||
// recovery would yield an empty tree → both buckets empty → filter ignored.
|
||||
name: "lone paren rejected",
|
||||
query: ")",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "unbalanced parens rejected",
|
||||
query: "((",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "bare operator rejected",
|
||||
query: "AND",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
// lexer-level error: recovery drops the whole expression.
|
||||
name: "unterminated quote rejected",
|
||||
query: "'unterminated",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
// recovery would drop only the malformed atom and keep the rest — a
|
||||
// partially applied filter with no error.
|
||||
name: "garbage atom alongside valid agg rejected",
|
||||
query: ") AND completion_tokens > 5",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "missing value rejected",
|
||||
query: "completion_tokens >",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
span, having, err := SplitFilterForAggregates(c.query, agg)
|
||||
if c.wantErr {
|
||||
require.Error(t, err)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, c.span, span, "span part")
|
||||
require.Equal(t, c.having, having, "having part")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,19 @@ func NewHavingExpressionRewriter() *HavingExpressionRewriter {
|
||||
}
|
||||
}
|
||||
|
||||
// Rewrite rewrites and validates a HAVING expression against a caller-supplied
|
||||
// column map (user-facing name -> SQL identifier/expression). Values are inlined, so
|
||||
// the result is a bare SQL boolean expression with no bound args. Used by callers
|
||||
// that project their own aggregate columns (e.g. the AI trace list) rather than the
|
||||
// query's Aggregations.
|
||||
func (r *HavingExpressionRewriter) Rewrite(expression string, columnMap map[string]string) (string, error) {
|
||||
if len(strings.TrimSpace(expression)) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
r.columnMap = columnMap
|
||||
return r.rewriteAndValidate(expression)
|
||||
}
|
||||
|
||||
// RewriteForTraces rewrites and validates the HAVING expression for a traces query.
|
||||
func (r *HavingExpressionRewriter) RewriteForTraces(expression string, aggregations []qbtypes.TraceAggregation) (string, error) {
|
||||
if len(strings.TrimSpace(expression)) == 0 {
|
||||
|
||||
@@ -82,6 +82,10 @@ func resourcesForQuery(query gjson.Result, variables map[string]qbtypes.Variable
|
||||
switch queryType {
|
||||
case qbtypes.QueryTypeBuilder.StringValue(), qbtypes.QueryTypeSubQuery.StringValue():
|
||||
return resourcesForBuilderQuery(queryType, query.Get("spec"), variables)
|
||||
case qbtypes.QueryTypeBuilderAI.StringValue():
|
||||
// An AI builder query is always a traces query; the signal is implied by the
|
||||
// type (and may be absent from the payload), so pin the resource directly.
|
||||
return builderQueryResourceRefs(queryType, coretypes.ResourceTelemetryResourceTraces, query.Get("spec"), variables)
|
||||
case qbtypes.QueryTypePromQL.StringValue():
|
||||
return []coretypes.ResourceWithID{{Resource: coretypes.ResourceTelemetryResourceMetrics, ID: typeWildcard}}, nil
|
||||
case qbtypes.QueryTypeClickHouseSQL.StringValue():
|
||||
@@ -103,7 +107,10 @@ func resourcesForBuilderQuery(queryType string, spec gjson.Result, variables map
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return builderQueryResourceRefs(queryType, resource, spec, variables)
|
||||
}
|
||||
|
||||
func builderQueryResourceRefs(queryType string, resource coretypes.Resource, spec gjson.Result, variables map[string]qbtypes.VariableItem) ([]coretypes.ResourceWithID, error) {
|
||||
ids, err := builderQuerySelectors(queryType, spec.Get("filter.expression").String(), variables)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -92,6 +92,20 @@ func TestQueryRangeResources(t *testing.T) {
|
||||
{Resource: coretypes.ResourceTelemetryResourceAuditLogs, ID: "builder_query/signoz.workspace.key.id/a"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ai builder query maps to traces resource without a signal",
|
||||
body: `{"compositeQuery":{"queries":[{"type":"builder_ai_query","spec":{"filter":{"expression":"signoz.workspace.key.id = 'checkout'"}}}]}}`,
|
||||
expected: []coretypes.ResourceWithID{
|
||||
{Resource: coretypes.ResourceTelemetryResourceTraces, ID: "builder_ai_query/signoz.workspace.key.id/checkout"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ai builder query without filter is wildcard",
|
||||
body: `{"compositeQuery":{"queries":[{"type":"builder_ai_query","spec":{}}]}}`,
|
||||
expected: []coretypes.ResourceWithID{
|
||||
{Resource: coretypes.ResourceTelemetryResourceTraces, ID: "builder_ai_query/*"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "promql is wildcard only",
|
||||
body: `{"compositeQuery":{"queries":[{"type":"promql","spec":{"query":"up"}}]}}`,
|
||||
|
||||
@@ -39,6 +39,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/sqlmigrator"
|
||||
"github.com/SigNoz/signoz/pkg/sqlschema"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statsreporter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/tokenizer"
|
||||
@@ -97,6 +98,9 @@ type Config struct {
|
||||
// Querier config
|
||||
Querier querier.Config `mapstructure:"querier"`
|
||||
|
||||
// StatementBuilder config
|
||||
StatementBuilder statementbuilder.Config `mapstructure:"statementbuilder"`
|
||||
|
||||
// Ruler config
|
||||
Ruler ruler.Config `mapstructure:"ruler"`
|
||||
|
||||
@@ -166,6 +170,7 @@ func NewConfig(ctx context.Context, logger *slog.Logger, resolverConfig config.R
|
||||
prometheus.NewConfigFactory(),
|
||||
alertmanager.NewConfigFactory(),
|
||||
querier.NewConfigFactory(),
|
||||
statementbuilder.NewConfigFactory(),
|
||||
ruler.NewConfigFactory(),
|
||||
emailing.NewConfigFactory(),
|
||||
sharder.NewConfigFactory(),
|
||||
|
||||
@@ -69,6 +69,8 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/version"
|
||||
"github.com/SigNoz/signoz/pkg/web"
|
||||
"github.com/SigNoz/signoz/pkg/web/noopweb"
|
||||
@@ -227,6 +229,7 @@ func NewSQLMigrationProviderFactories(
|
||||
sqlmigration.NewAddTelemetryTuplesFactory(sqlstore),
|
||||
sqlmigration.NewAddTagRelationRankFactory(sqlstore, sqlschema),
|
||||
sqlmigration.NewMigrateDashboardsV1ToV2Factory(sqlstore, sqlschema, dashboardStore, tagModule),
|
||||
sqlmigration.NewFillDashboardMeterSourceFactory(sqlstore, dashboardStore),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -285,9 +288,9 @@ func NewStatsReporterProviderFactories(aggregator statsreporter.Aggregator, orgG
|
||||
)
|
||||
}
|
||||
|
||||
func NewQuerierProviderFactories(telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, cache cache.Cache, flagger flagger.Flagger) factory.NamedMap[factory.ProviderFactory[querier.Querier, querier.Config]] {
|
||||
func NewQuerierProviderFactories(telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, metadataStore telemetrytypes.MetadataStore, traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation], aiTraceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation], logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation], auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation], metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation], meterStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation], traceOperatorStmtBuilder qbtypes.TraceOperatorStatementBuilder, bucketCache querier.BucketCache, flagger flagger.Flagger) factory.NamedMap[factory.ProviderFactory[querier.Querier, querier.Config]] {
|
||||
return factory.MustNewNamedMap(
|
||||
signozquerier.NewFactory(telemetryStore, prometheus, cache, flagger),
|
||||
signozquerier.NewFactory(telemetryStore, prometheus, metadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -48,16 +48,18 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/sqlmigrator"
|
||||
"github.com/SigNoz/signoz/pkg/sqlschema"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/aistatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/auditstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/logsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/meterstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/tracesstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statsreporter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryaudit"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetadata"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
pkgtokenizer "github.com/SigNoz/signoz/pkg/tokenizer"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/version"
|
||||
"github.com/SigNoz/signoz/pkg/zeus"
|
||||
@@ -96,6 +98,67 @@ type SigNoz struct {
|
||||
MeterReporter meterreporter.Reporter
|
||||
}
|
||||
|
||||
// newQueryStack assembles the query stack once and returns, in order: the shared
|
||||
// telemetry metadata store (reused elsewhere in signoz.New), the per-signal
|
||||
// statement builders (trace, ai-trace, log, audit, metric, meter, trace-operator),
|
||||
// and the bucket cache. It is the only place that imports the concrete
|
||||
// statement-builder sub-packages.
|
||||
func newQueryStack(
|
||||
ctx context.Context,
|
||||
settings factory.ProviderSettings,
|
||||
config Config,
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
cache cache.Cache,
|
||||
fl flagger.Flagger,
|
||||
) (
|
||||
telemetrytypes.MetadataStore,
|
||||
qbtypes.StatementBuilder[qbtypes.TraceAggregation],
|
||||
qbtypes.StatementBuilder[qbtypes.TraceAggregation],
|
||||
qbtypes.StatementBuilder[qbtypes.LogAggregation],
|
||||
qbtypes.StatementBuilder[qbtypes.LogAggregation],
|
||||
qbtypes.StatementBuilder[qbtypes.MetricAggregation],
|
||||
qbtypes.StatementBuilder[qbtypes.MetricAggregation],
|
||||
qbtypes.TraceOperatorStatementBuilder,
|
||||
querier.BucketCache,
|
||||
error,
|
||||
) {
|
||||
metadataStore := telemetrymetadata.NewTelemetryMetaStore(settings, telemetryStore, fl)
|
||||
|
||||
cfg := config.StatementBuilder
|
||||
traceStmtBuilder, err := tracesstatementbuilder.NewFactory(telemetryStore, metadataStore, fl).New(ctx, settings, cfg)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
|
||||
}
|
||||
aiTraceStmtBuilder, err := aistatementbuilder.NewFactory(telemetryStore, metadataStore, fl).New(ctx, settings, cfg)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
|
||||
}
|
||||
traceOperatorStmtBuilder, err := tracesstatementbuilder.NewOperatorFactory(telemetryStore, metadataStore, fl).New(ctx, settings, cfg)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
|
||||
}
|
||||
logStmtBuilder, err := logsstatementbuilder.NewFactory(telemetryStore, metadataStore, fl).New(ctx, settings, cfg)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
|
||||
}
|
||||
auditStmtBuilder, err := auditstatementbuilder.NewFactory(metadataStore, fl).New(ctx, settings, cfg)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
|
||||
}
|
||||
metricStmtBuilder, err := metricsstatementbuilder.NewFactory(metadataStore, fl).New(ctx, settings, cfg)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
|
||||
}
|
||||
meterStmtBuilder, err := meterstatementbuilder.NewFactory(metadataStore, fl).New(ctx, settings, cfg)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
bucketCache := querier.NewBucketCache(settings, cache, config.Querier.CacheTTL, config.Querier.FluxInterval)
|
||||
|
||||
return metadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, nil
|
||||
}
|
||||
|
||||
func New(
|
||||
ctx context.Context,
|
||||
config Config,
|
||||
@@ -254,12 +317,19 @@ func New(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Assemble the query stack (metadata store, statement builders, bucket cache) once,
|
||||
// and reuse the single metadata store everywhere downstream.
|
||||
telemetryMetadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, err := newQueryStack(ctx, providerSettings, config, telemetrystore, cache, flagger)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Initialize querier from the available querier provider factories
|
||||
querier, err := factory.NewProviderFromNamedMap(
|
||||
ctx,
|
||||
providerSettings,
|
||||
config.Querier,
|
||||
NewQuerierProviderFactories(telemetrystore, prometheus, cache, flagger),
|
||||
NewQuerierProviderFactories(telemetrystore, prometheus, telemetryMetadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger),
|
||||
config.Querier.Provider(),
|
||||
)
|
||||
if err != nil {
|
||||
@@ -427,35 +497,6 @@ func New(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Initialize telemetry metadata store
|
||||
// TODO: consolidate other telemetrymetadata.NewTelemetryMetaStore initializations to reuse this instance instead.
|
||||
telemetryMetadataStore := telemetrymetadata.NewTelemetryMetaStore(
|
||||
providerSettings,
|
||||
telemetrystore,
|
||||
telemetrytraces.DBName,
|
||||
telemetrytraces.TagAttributesV2TableName,
|
||||
telemetrytraces.SpanAttributesKeysTblName,
|
||||
telemetrytraces.SpanIndexV3TableName,
|
||||
telemetrymetrics.DBName,
|
||||
telemetrymetrics.AttributesMetadataTableName,
|
||||
telemetrymeter.DBName,
|
||||
telemetrymeter.SamplesAgg1dTableName,
|
||||
telemetrylogs.DBName,
|
||||
telemetrylogs.LogsV2TableName,
|
||||
telemetrylogs.TagAttributesV2TableName,
|
||||
telemetrylogs.LogAttributeKeysTblName,
|
||||
telemetrylogs.LogResourceKeysTblName,
|
||||
telemetryaudit.DBName,
|
||||
telemetryaudit.AuditLogsTableName,
|
||||
telemetryaudit.TagAttributesTableName,
|
||||
telemetryaudit.LogAttributeKeysTblName,
|
||||
telemetryaudit.LogResourceKeysTblName,
|
||||
telemetrymetadata.DBName,
|
||||
telemetrymetadata.AttributesMetadataTableName,
|
||||
telemetrymetadata.ColumnEvolutionMetadataTableName,
|
||||
flagger,
|
||||
)
|
||||
|
||||
global, err := factory.NewProviderFromNamedMap(
|
||||
ctx,
|
||||
providerSettings,
|
||||
|
||||
159
pkg/sqlmigration/104_fill_dashboard_meter_source.go
Normal file
159
pkg/sqlmigration/104_fill_dashboard_meter_source.go
Normal file
@@ -0,0 +1,159 @@
|
||||
package sqlmigration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
qb "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/uptrace/bun"
|
||||
"github.com/uptrace/bun/migrate"
|
||||
)
|
||||
|
||||
// meterMetricNames are the SigNoz meter metrics; a metrics query aggregating one is a
|
||||
// meter query.
|
||||
var meterMetricNames = map[string]bool{
|
||||
"signoz.meter.log.count": true,
|
||||
"signoz.meter.log.size": true,
|
||||
"signoz.meter.span.count": true,
|
||||
"signoz.meter.span.size": true,
|
||||
"signoz.meter.metric.datapoint.count": true,
|
||||
"signoz.meter.platform.active": true,
|
||||
}
|
||||
|
||||
type fillDashboardMeterSource struct {
|
||||
sqlstore sqlstore.SQLStore
|
||||
dashboardStore dashboardtypes.Store
|
||||
settings factory.ProviderSettings
|
||||
}
|
||||
|
||||
func NewFillDashboardMeterSourceFactory(sqlstore sqlstore.SQLStore, dashboardStore dashboardtypes.Store) factory.ProviderFactory[SQLMigration, Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("fill_dashboard_meter_source"),
|
||||
func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
|
||||
return &fillDashboardMeterSource{sqlstore: sqlstore, dashboardStore: dashboardStore, settings: ps}, nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (migration *fillDashboardMeterSource) Register(migrations *migrate.Migrations) error {
|
||||
return migrations.Register(migration.Up, migration.Down)
|
||||
}
|
||||
|
||||
// Up resets the meter source that 103 dropped when converting v1 meter queries to v2:
|
||||
// every metrics query aggregating a meter metric is set back to source "meter". One
|
||||
// transaction; v1 (unparseable-as-v2) dashboards are skipped.
|
||||
func (migration *fillDashboardMeterSource) Up(ctx context.Context, _ *bun.DB) error {
|
||||
return migration.sqlstore.RunInTxCtx(ctx, nil, func(ctx context.Context) error {
|
||||
var orgIDs []string
|
||||
if err := migration.sqlstore.BunDBCtx(ctx).NewSelect().Model((*types.Organization)(nil)).Column("id").Scan(ctx, &orgIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, id := range orgIDs {
|
||||
orgID, err := valuer.NewUUID(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := migration.fillOrg(ctx, orgID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// fillOrg fills the meter source on every v2 dashboard in the org that needs it. Runs
|
||||
// inside the caller's transaction.
|
||||
func (migration *fillDashboardMeterSource) fillOrg(ctx context.Context, orgID valuer.UUID) error {
|
||||
// List, not ListV2: ListV2 paginates and excludes system dashboards; a migration needs every row.
|
||||
storables, err := migration.dashboardStore.List(ctx, orgID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger := migration.settings.Logger
|
||||
var stillInV1, skippedNoMeter, wouldMigrateButErrored, migrated int
|
||||
for _, storable := range storables {
|
||||
// ToDashboardV2 rejects non-v2 data, so v1 dashboards fall through untouched.
|
||||
dashboard, err := storable.ToDashboardV2(nil)
|
||||
if err != nil {
|
||||
stillInV1++
|
||||
continue
|
||||
}
|
||||
if !fillMeterSource(&dashboard.Spec) {
|
||||
skippedNoMeter++
|
||||
continue
|
||||
}
|
||||
restorable, err := dashboard.ToStorableDashboard()
|
||||
if err != nil {
|
||||
wouldMigrateButErrored++
|
||||
logger.WarnContext(ctx, "failed to re-serialize dashboard after filling meter source", slog.String("org_id", orgID.String()), slog.String("dashboard_id", storable.ID.String()), slog.Any("error", err))
|
||||
continue
|
||||
}
|
||||
if err := migration.dashboardStore.Update(ctx, orgID, restorable); err != nil {
|
||||
return err
|
||||
}
|
||||
migrated++
|
||||
}
|
||||
|
||||
logger.InfoContext(ctx, "filled meter source on v2 dashboards",
|
||||
slog.String("org_id", orgID.String()),
|
||||
slog.Int("total", len(storables)),
|
||||
slog.Int("still_in_v1", stillInV1),
|
||||
slog.Int("skipped_no_meter", skippedNoMeter),
|
||||
slog.Int("would_migrate_but_errored", wouldMigrateButErrored),
|
||||
slog.Int("migrated", migrated),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
// fillMeterSource sets Source=meter on every metrics builder query in the spec whose
|
||||
// aggregation targets a meter metric, reporting whether anything changed.
|
||||
func fillMeterSource(spec *dashboardtypes.DashboardSpec) bool {
|
||||
changed := false
|
||||
for _, panel := range spec.Panels {
|
||||
if panel == nil {
|
||||
continue
|
||||
}
|
||||
for _, query := range panel.Spec.Queries {
|
||||
switch plugin := query.Spec.Plugin.Spec.(type) {
|
||||
case *qb.CompositeQuery:
|
||||
for i := range plugin.Queries {
|
||||
changed = setMeterSource(&plugin.Queries[i].Spec) || changed
|
||||
}
|
||||
case *dashboardtypes.BuilderQuerySpec:
|
||||
changed = setMeterSource(&plugin.Spec) || changed
|
||||
}
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// setMeterSource sets source "meter" on a metrics query aggregating a meter metric. The
|
||||
// type assertion doubles as the metrics-signal check; the already-meter guard keeps it
|
||||
// idempotent. Written back through the pointer, as an interface value isn't addressable.
|
||||
func setMeterSource(spec *any) bool {
|
||||
query, ok := (*spec).(qb.QueryBuilderQuery[qb.MetricAggregation])
|
||||
if !ok || query.Source == telemetrytypes.SourceMeter {
|
||||
return false
|
||||
}
|
||||
for _, aggregation := range query.Aggregations {
|
||||
if meterMetricNames[aggregation.MetricName] {
|
||||
query.Source = telemetrytypes.SourceMeter
|
||||
*spec = query
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (migration *fillDashboardMeterSource) Down(context.Context, *bun.DB) error {
|
||||
return nil
|
||||
}
|
||||
84
pkg/statementbuilder/aistatementbuilder/statement_builder.go
Normal file
84
pkg/statementbuilder/aistatementbuilder/statement_builder.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package aistatementbuilder
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
scopedtraces "github.com/SigNoz/signoz/pkg/statementbuilder/scopedtracesstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
// NewFactory returns a provider factory for the AI trace statement builder
|
||||
// (builder_ai_query): the gen_ai Scope paired with the domain-neutral scoped-trace
|
||||
// topology, which owns the query construction.
|
||||
//
|
||||
// The gen_ai gate/column keys are surfaced by the metadata store itself
|
||||
// (enrichWithGenAIKeys), so queries work before any gen_ai metadata is ingested.
|
||||
func NewFactory(
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
fl flagger.Flagger,
|
||||
) factory.ProviderFactory[qbtypes.StatementBuilder[qbtypes.TraceAggregation], statementbuilder.Config] {
|
||||
return scopedtraces.NewFactory(factory.MustNewName("ai"), Scope(), telemetryStore, metadataStore, fl)
|
||||
}
|
||||
|
||||
// Scope describes gen_ai for the scoped trace builder: an AI trace has >=1 gen_ai
|
||||
// LLM, tool, or agent span, and its list adds AI/LLM per-trace metrics on top of the
|
||||
// common columns. This package holds only gen_ai domain knowledge; the query
|
||||
// topology lives in scopedtracesstatementbuilder.
|
||||
func Scope() scopedtraces.TraceScope {
|
||||
gateKeyNames := []string{telemetrytypes.GenAIRequestModel, telemetrytypes.GenAIToolName, telemetrytypes.GenAIAgentName}
|
||||
gateExprs := make([]string, 0, len(gateKeyNames))
|
||||
gateKeys := make([]*telemetrytypes.TelemetryFieldKey, 0, len(gateKeyNames))
|
||||
for _, name := range gateKeyNames {
|
||||
gateExprs = append(gateExprs, name+" EXISTS")
|
||||
gateKeys = append(gateKeys, &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
})
|
||||
}
|
||||
|
||||
defs := telemetrytypes.GenAIFieldDefinitions
|
||||
reqModel := defs[telemetrytypes.GenAIRequestModel]
|
||||
toolName := defs[telemetrytypes.GenAIToolName]
|
||||
inTok := defs[telemetrytypes.GenAIUsageInputTokens]
|
||||
outTok := defs[telemetrytypes.GenAIUsageOutputTokens]
|
||||
cost := defs[telemetrytypes.SignozGenAITotalCost]
|
||||
inMsg := defs[telemetrytypes.GenAIInputMessages]
|
||||
outMsg := defs[telemetrytypes.GenAIOutputMessages]
|
||||
|
||||
str := telemetrytypes.FieldDataTypeString
|
||||
columns := append(scopedtraces.CommonTraceColumns(),
|
||||
// LLM calls only (request model present), not the full gate.
|
||||
scopedtraces.TraceColumn{Alias: "llm_call_count", Orderable: true, Expr: scopedtraces.CountExists(&reqModel)},
|
||||
scopedtraces.TraceColumn{Alias: "tool_call_count", Orderable: true, Expr: scopedtraces.CountExists(&toolName)},
|
||||
scopedtraces.TraceColumn{Alias: "distinct_tool_count", Orderable: true, Expr: scopedtraces.UniqCount(&toolName, str)},
|
||||
// tokens live only on LLM spans, so a plain sum needs no gate scoping.
|
||||
scopedtraces.TraceColumn{Alias: "input_tokens", Orderable: true, Expr: scopedtraces.Reduce(scopedtraces.AggSum, &inTok)},
|
||||
scopedtraces.TraceColumn{Alias: "output_tokens", Orderable: true, Expr: scopedtraces.Reduce(scopedtraces.AggSum, &outTok)},
|
||||
scopedtraces.TraceColumn{Alias: "total_tokens", Orderable: true, Expr: scopedtraces.SumOfKeys(telemetrytypes.FieldDataTypeFloat64, &inTok, &outTok)},
|
||||
// per-span cost attached by the SigNoz LLM pricing processor.
|
||||
scopedtraces.TraceColumn{Alias: "estimated_total_cost", Orderable: true, Expr: scopedtraces.Reduce(scopedtraces.AggSum, &cost)},
|
||||
// slowest single LLM call in the trace.
|
||||
scopedtraces.TraceColumn{Alias: "max_llm_duration_nano", Orderable: true, Expr: scopedtraces.ScopedToKeyColumn(scopedtraces.AggMax, scopedtraces.IntrinsicSpanKey("duration_nano"), &reqModel)},
|
||||
// errors across the whole trace (any span), so display-only.
|
||||
scopedtraces.TraceColumn{Alias: "error_count", Expr: scopedtraces.CondCount(scopedtraces.IntrinsicSpanKey("has_error"), qbtypes.FilterOperatorEqual, true)},
|
||||
// timestamp of the last gen_ai span (LLM/tool/agent), hence gate-scoped.
|
||||
scopedtraces.TraceColumn{Alias: "last_activity_time", Orderable: true, Expr: scopedtraces.ScopedReduce(scopedtraces.AggMax, scopedtraces.IntrinsicSpanKey("timestamp"))},
|
||||
// previews: first call's input (the prompt), last call's output (the answer).
|
||||
scopedtraces.TraceColumn{Alias: "input", SpanLevel: true, Expr: scopedtraces.PickBy(&inMsg, str, scopedtraces.IntrinsicSpanKey("timestamp"), scopedtraces.PickEarliest)},
|
||||
scopedtraces.TraceColumn{Alias: "output", SpanLevel: true, Expr: scopedtraces.PickBy(&outMsg, str, scopedtraces.IntrinsicSpanKey("timestamp"), scopedtraces.PickLatest)},
|
||||
)
|
||||
|
||||
return scopedtraces.TraceScope{
|
||||
FilterExpression: strings.Join(gateExprs, " OR "),
|
||||
FieldKeys: gateKeys,
|
||||
Columns: columns,
|
||||
DefaultOrderAlias: "last_activity_time",
|
||||
}
|
||||
}
|
||||
1106
pkg/statementbuilder/aistatementbuilder/statement_builder_test.go
Normal file
1106
pkg/statementbuilder/aistatementbuilder/statement_builder_test.go
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,4 @@
|
||||
package telemetryaudit
|
||||
package auditstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,7 +10,9 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryresourcefilter"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/resourcefilter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -29,6 +31,25 @@ type auditQueryStatementBuilder struct {
|
||||
|
||||
var _ qbtypes.StatementBuilder[qbtypes.LogAggregation] = (*auditQueryStatementBuilder)(nil)
|
||||
|
||||
// NewFactory returns a provider factory for the audit statement builder. Its New
|
||||
// internalizes the FieldMapper, ConditionBuilder, and AggExprRewriter.
|
||||
func NewFactory(
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
fl flagger.Flagger,
|
||||
) factory.ProviderFactory[qbtypes.StatementBuilder[qbtypes.LogAggregation], statementbuilder.Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("audit"),
|
||||
func(_ context.Context, settings factory.ProviderSettings, _ statementbuilder.Config) (qbtypes.StatementBuilder[qbtypes.LogAggregation], error) {
|
||||
fm := audittelemetryschema.NewFieldMapper()
|
||||
cb := audittelemetryschema.NewConditionBuilder(fm)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(settings, audittelemetryschema.DefaultFullTextColumn, fm, cb, fl)
|
||||
return NewAuditQueryStatementBuilder(
|
||||
settings, metadataStore, fm, cb, aggExprRewriter, audittelemetryschema.DefaultFullTextColumn, fl,
|
||||
), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func NewAuditQueryStatementBuilder(
|
||||
settings factory.ProviderSettings,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
@@ -38,12 +59,12 @@ func NewAuditQueryStatementBuilder(
|
||||
fullTextColumn *telemetrytypes.TelemetryFieldKey,
|
||||
flagger flagger.Flagger,
|
||||
) *auditQueryStatementBuilder {
|
||||
auditSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryaudit")
|
||||
auditSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema")
|
||||
|
||||
resourceFilterStmtBuilder := telemetryresourcefilter.New[qbtypes.LogAggregation](
|
||||
resourceFilterStmtBuilder := resourcefilter.New[qbtypes.LogAggregation](
|
||||
settings,
|
||||
DBName,
|
||||
LogsResourceTableName,
|
||||
audittelemetryschema.DBName,
|
||||
audittelemetryschema.LogsResourceTableName,
|
||||
telemetrytypes.SignalLogs,
|
||||
telemetrytypes.SourceAudit,
|
||||
metadataStore,
|
||||
@@ -191,8 +212,8 @@ func (b *auditQueryStatementBuilder) adjustKeys(ctx context.Context, keys map[st
|
||||
}
|
||||
|
||||
func (b *auditQueryStatementBuilder) adjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) []string {
|
||||
if _, ok := IntrinsicFields[key.Name]; ok {
|
||||
intrinsicField := IntrinsicFields[key.Name]
|
||||
if _, ok := audittelemetryschema.IntrinsicFields[key.Name]; ok {
|
||||
intrinsicField := audittelemetryschema.IntrinsicFields[key.Name]
|
||||
return querybuilder.AdjustKey(key, keys, &intrinsicField)
|
||||
}
|
||||
return querybuilder.AdjustKey(key, keys, nil)
|
||||
@@ -219,26 +240,26 @@ func (b *auditQueryStatementBuilder) buildListQuery(
|
||||
cteArgs = append(cteArgs, args)
|
||||
}
|
||||
|
||||
sb.Select(TimestampColumn)
|
||||
sb.SelectMore(IDColumn)
|
||||
sb.Select(audittelemetryschema.TimestampColumn)
|
||||
sb.SelectMore(audittelemetryschema.IDColumn)
|
||||
if len(query.SelectFields) == 0 {
|
||||
sb.SelectMore(TraceIDColumn)
|
||||
sb.SelectMore(SpanIDColumn)
|
||||
sb.SelectMore(TraceFlagsColumn)
|
||||
sb.SelectMore(SeverityTextColumn)
|
||||
sb.SelectMore(SeverityNumberColumn)
|
||||
sb.SelectMore(ScopeNameColumn)
|
||||
sb.SelectMore(ScopeVersionColumn)
|
||||
sb.SelectMore(BodyColumn)
|
||||
sb.SelectMore(EventNameColumn)
|
||||
sb.SelectMore(AttributesStringColumn)
|
||||
sb.SelectMore(AttributesNumberColumn)
|
||||
sb.SelectMore(AttributesBoolColumn)
|
||||
sb.SelectMore(ResourceColumn)
|
||||
sb.SelectMore(ScopeStringColumn)
|
||||
sb.SelectMore(audittelemetryschema.TraceIDColumn)
|
||||
sb.SelectMore(audittelemetryschema.SpanIDColumn)
|
||||
sb.SelectMore(audittelemetryschema.TraceFlagsColumn)
|
||||
sb.SelectMore(audittelemetryschema.SeverityTextColumn)
|
||||
sb.SelectMore(audittelemetryschema.SeverityNumberColumn)
|
||||
sb.SelectMore(audittelemetryschema.ScopeNameColumn)
|
||||
sb.SelectMore(audittelemetryschema.ScopeVersionColumn)
|
||||
sb.SelectMore(audittelemetryschema.BodyColumn)
|
||||
sb.SelectMore(audittelemetryschema.EventNameColumn)
|
||||
sb.SelectMore(audittelemetryschema.AttributesStringColumn)
|
||||
sb.SelectMore(audittelemetryschema.AttributesNumberColumn)
|
||||
sb.SelectMore(audittelemetryschema.AttributesBoolColumn)
|
||||
sb.SelectMore(audittelemetryschema.ResourceColumn)
|
||||
sb.SelectMore(audittelemetryschema.ScopeStringColumn)
|
||||
} else {
|
||||
for index := range query.SelectFields {
|
||||
if query.SelectFields[index].Name == TimestampColumn || query.SelectFields[index].Name == IDColumn {
|
||||
if query.SelectFields[index].Name == audittelemetryschema.TimestampColumn || query.SelectFields[index].Name == audittelemetryschema.IDColumn {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -250,7 +271,7 @@ func (b *auditQueryStatementBuilder) buildListQuery(
|
||||
}
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, AuditLogsTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", audittelemetryschema.DBName, audittelemetryschema.AuditLogsTableName))
|
||||
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables)
|
||||
if err != nil {
|
||||
@@ -340,7 +361,7 @@ func (b *auditQueryStatementBuilder) buildTimeSeriesQuery(
|
||||
sb.SelectMore(fmt.Sprintf("%s AS __result_%d", rewritten, i))
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, AuditLogsTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", audittelemetryschema.DBName, audittelemetryschema.AuditLogsTableName))
|
||||
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables)
|
||||
if err != nil {
|
||||
@@ -478,7 +499,7 @@ func (b *auditQueryStatementBuilder) buildScalarQuery(
|
||||
}
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, AuditLogsTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", audittelemetryschema.DBName, audittelemetryschema.AuditLogsTableName))
|
||||
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables)
|
||||
if err != nil {
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetryaudit
|
||||
package auditstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
||||
@@ -54,8 +55,8 @@ func newTestAuditStatementBuilder(t *testing.T) *auditQueryStatementBuilder {
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = auditFieldKeyMap()
|
||||
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fm := audittelemetryschema.NewFieldMapper()
|
||||
cb := audittelemetryschema.NewConditionBuilder(fm)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
return NewAuditQueryStatementBuilder(
|
||||
@@ -64,7 +65,7 @@ func newTestAuditStatementBuilder(t *testing.T) *auditQueryStatementBuilder {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
audittelemetryschema.DefaultFullTextColumn,
|
||||
fl,
|
||||
)
|
||||
}
|
||||
41
pkg/statementbuilder/config.go
Normal file
41
pkg/statementbuilder/config.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package statementbuilder
|
||||
|
||||
import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
)
|
||||
|
||||
// SkipResourceFingerprint configures when the resource fingerprint subquery is skipped in favor of main-table filtering.
|
||||
type SkipResourceFingerprint struct {
|
||||
Enabled bool `yaml:"enabled" mapstructure:"enabled"`
|
||||
// If count of fingerprint is above threshold, skip the fingerprint subquery and filter on main table instead.
|
||||
Threshold uint64 `yaml:"threshold" mapstructure:"threshold"`
|
||||
}
|
||||
|
||||
// Config represents the configuration for the statement builders.
|
||||
type Config struct {
|
||||
// SkipResourceFingerprint configures when the resource fingerprint subquery is skipped in favor of main-table filtering.
|
||||
SkipResourceFingerprint SkipResourceFingerprint `yaml:"skip_resource_fingerprint" mapstructure:"skip_resource_fingerprint"`
|
||||
}
|
||||
|
||||
// NewConfigFactory creates a new config factory for the statement builders.
|
||||
func NewConfigFactory() factory.ConfigFactory {
|
||||
return factory.NewConfigFactory(factory.MustNewName("statementbuilder"), newConfig)
|
||||
}
|
||||
|
||||
func newConfig() factory.Config {
|
||||
return Config{
|
||||
SkipResourceFingerprint: SkipResourceFingerprint{
|
||||
Enabled: false,
|
||||
Threshold: 100000,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Validate validates the configuration.
|
||||
func (c Config) Validate() error {
|
||||
if c.SkipResourceFingerprint.Enabled && c.SkipResourceFingerprint.Threshold == 0 {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "skip_resource_fingerprint.threshold must be > 0 when enabled")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrylogs
|
||||
package logsstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
||||
@@ -43,14 +44,14 @@ func TestStatementBuilderGroupByUnknownKey(t *testing.T) {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
fl := flaggertest.WithUseJSONBody(t, c.useJSONBody)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
statementBuilder := NewLogQueryStatementBuilder(
|
||||
instrumentationtest.New().ToProviderSettings(),
|
||||
mockMetadataStore, fm, cb, aggExprRewriter,
|
||||
DefaultFullTextColumn, fl, nil, false, 100000,
|
||||
logstelemetryschema.DefaultFullTextColumn, fl, nil, false, 100000,
|
||||
)
|
||||
|
||||
query := qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrylogs
|
||||
package logsstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
||||
@@ -1225,7 +1226,7 @@ func TestJSONStmtBuilder_OrderBy(t *testing.T) {
|
||||
func TestResourceAggrAndGroupBy_WithJSONEnabled(t *testing.T) {
|
||||
statementBuilder, metadataStore := buildJSONTestStatementBuilder(t, false)
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keysMap := buildCompleteFieldKeyMap(releaseTime)
|
||||
keysMap := logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
for _, keys := range keysMap {
|
||||
for _, key := range keys {
|
||||
metadataStore.SetKey(key)
|
||||
@@ -1287,7 +1288,7 @@ func TestResourceAggrAndGroupBy_WithJSONEnabled(t *testing.T) {
|
||||
|
||||
func buildTestTelemetryMetadataStore(t *testing.T, addIndexes bool) *telemetrytypestest.MockMetadataStore {
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.SetStaticFields(IntrinsicFields)
|
||||
mockMetadataStore.SetStaticFields(logstelemetryschema.IntrinsicFields)
|
||||
types, _ := telemetrytypes.TestJSONTypeSet()
|
||||
for path, fieldDataTypes := range types {
|
||||
for _, fdt := range fieldDataTypes {
|
||||
@@ -1307,15 +1308,15 @@ func buildTestTelemetryMetadataStore(t *testing.T, addIndexes bool) *telemetryty
|
||||
Name: path,
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
FieldDataType: fdt,
|
||||
BaseColumn: LogsV2BodyV2Column,
|
||||
IndexExpression: schemamigrator.JSONSubColumnIndexExpr(LogsV2BodyV2Column, path, jsonType.StringValue()),
|
||||
BaseColumn: logstelemetryschema.LogsV2BodyV2Column,
|
||||
IndexExpression: schemamigrator.JSONSubColumnIndexExpr(logstelemetryschema.LogsV2BodyV2Column, path, jsonType.StringValue()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
err := key.SetJSONAccessPlan(telemetrytypes.JSONColumnMetadata{
|
||||
BaseColumn: LogsV2BodyV2Column,
|
||||
PromotedColumn: LogsV2BodyPromotedColumn,
|
||||
BaseColumn: logstelemetryschema.LogsV2BodyV2Column,
|
||||
PromotedColumn: logstelemetryschema.LogsV2BodyPromotedColumn,
|
||||
}, types)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -1331,8 +1332,8 @@ func buildJSONTestStatementBuilder(t *testing.T, addIndexes bool) (*logQueryStat
|
||||
|
||||
mockMetadataStore := buildTestTelemetryMetadataStore(t, addIndexes)
|
||||
fl := flaggertest.WithUseJSONBody(t, true)
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
@@ -1342,7 +1343,7 @@ func buildJSONTestStatementBuilder(t *testing.T, addIndexes bool) (*logQueryStat
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrylogs
|
||||
package logsstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,7 +10,9 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryresourcefilter"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/resourcefilter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -19,12 +21,22 @@ import (
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
)
|
||||
|
||||
const bodySearchDefaultWarning = "body searches default to `body.message:string`. Use `body.<key>` to search a different field inside body"
|
||||
|
||||
func bodyAliasExpression(bodyJSONEnabled bool) string {
|
||||
if !bodyJSONEnabled {
|
||||
return logstelemetryschema.LogsV2BodyColumn
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s as body", logstelemetryschema.LogsV2BodyV2Column)
|
||||
}
|
||||
|
||||
type logQueryStatementBuilder struct {
|
||||
logger *slog.Logger
|
||||
metadataStore telemetrytypes.MetadataStore
|
||||
fm qbtypes.FieldMapper
|
||||
cb qbtypes.ConditionBuilder
|
||||
resourceFilterResolver *telemetryresourcefilter.ResourceFingerprintResolver[qbtypes.LogAggregation]
|
||||
resourceFilterResolver *resourcefilter.ResourceFingerprintResolver[qbtypes.LogAggregation]
|
||||
aggExprRewriter qbtypes.AggExprRewriter
|
||||
fl flagger.Flagger
|
||||
skipResourceFingerprintEnabled bool
|
||||
@@ -34,6 +46,28 @@ type logQueryStatementBuilder struct {
|
||||
|
||||
var _ qbtypes.StatementBuilder[qbtypes.LogAggregation] = (*logQueryStatementBuilder)(nil)
|
||||
|
||||
// NewFactory returns a provider factory for the logs statement builder. Its New
|
||||
// internalizes the FieldMapper, ConditionBuilder, and AggExprRewriter, and reads
|
||||
// SkipResourceFingerprint from the config.
|
||||
func NewFactory(
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
fl flagger.Flagger,
|
||||
) factory.ProviderFactory[qbtypes.StatementBuilder[qbtypes.LogAggregation], statementbuilder.Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("logs"),
|
||||
func(_ context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.StatementBuilder[qbtypes.LogAggregation], error) {
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(settings, logstelemetryschema.DefaultFullTextColumn, fm, cb, fl)
|
||||
return NewLogQueryStatementBuilder(
|
||||
settings, metadataStore, fm, cb, aggExprRewriter, logstelemetryschema.DefaultFullTextColumn,
|
||||
fl, telemetryStore, cfg.SkipResourceFingerprint.Enabled, cfg.SkipResourceFingerprint.Threshold,
|
||||
), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func NewLogQueryStatementBuilder(
|
||||
settings factory.ProviderSettings,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
@@ -46,12 +80,12 @@ func NewLogQueryStatementBuilder(
|
||||
skipResourceFingerprintEnable bool,
|
||||
skipResourceFingerprintThreshold uint64,
|
||||
) *logQueryStatementBuilder {
|
||||
logsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrylogs")
|
||||
logsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema")
|
||||
|
||||
resourceFilterResolver := telemetryresourcefilter.NewResolver[qbtypes.LogAggregation](
|
||||
resourceFilterResolver := resourcefilter.NewResolver[qbtypes.LogAggregation](
|
||||
settings,
|
||||
DBName,
|
||||
LogsResourceV2TableName,
|
||||
logstelemetryschema.DBName,
|
||||
logstelemetryschema.LogsResourceV2TableName,
|
||||
telemetrytypes.SignalLogs,
|
||||
telemetrytypes.SourceUnspecified,
|
||||
metadataStore,
|
||||
@@ -174,7 +208,7 @@ func getKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation], bo
|
||||
// TODO(Piyush): Setup better for coming FTS support.
|
||||
if bodyJSONEnabled {
|
||||
for _, sel := range keySelectors {
|
||||
if sel.Name == LogsV2BodyColumn {
|
||||
if sel.Name == logstelemetryschema.LogsV2BodyColumn {
|
||||
warnings = append(warnings, bodySearchDefaultWarning)
|
||||
break
|
||||
}
|
||||
@@ -250,8 +284,8 @@ func (b *logQueryStatementBuilder) adjustKeys(ctx context.Context, keys map[stri
|
||||
func (b *logQueryStatementBuilder) adjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) []string {
|
||||
// First check if it matches with any intrinsic fields
|
||||
var intrinsicOrCalculatedField telemetrytypes.TelemetryFieldKey
|
||||
if _, ok := IntrinsicFields[key.Name]; ok {
|
||||
intrinsicOrCalculatedField = IntrinsicFields[key.Name]
|
||||
if _, ok := logstelemetryschema.IntrinsicFields[key.Name]; ok {
|
||||
intrinsicOrCalculatedField = logstelemetryschema.IntrinsicFields[key.Name]
|
||||
return querybuilder.AdjustKey(key, keys, &intrinsicOrCalculatedField)
|
||||
}
|
||||
|
||||
@@ -284,28 +318,28 @@ func (b *logQueryStatementBuilder) buildListQuery(
|
||||
}
|
||||
|
||||
// Select timestamp and id by default
|
||||
sb.Select(LogsV2TimestampColumn)
|
||||
sb.SelectMore(LogsV2IDColumn)
|
||||
sb.Select(logstelemetryschema.LogsV2TimestampColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2IDColumn)
|
||||
if len(query.SelectFields) == 0 {
|
||||
// Select all default columns
|
||||
sb.SelectMore(LogsV2TraceIDColumn)
|
||||
sb.SelectMore(LogsV2SpanIDColumn)
|
||||
sb.SelectMore(LogsV2TraceFlagsColumn)
|
||||
sb.SelectMore(LogsV2SeverityTextColumn)
|
||||
sb.SelectMore(LogsV2SeverityNumberColumn)
|
||||
sb.SelectMore(LogsV2ScopeNameColumn)
|
||||
sb.SelectMore(LogsV2ScopeVersionColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2TraceIDColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2SpanIDColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2TraceFlagsColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2SeverityTextColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2SeverityNumberColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2ScopeNameColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2ScopeVersionColumn)
|
||||
sb.SelectMore(bodyAliasExpression(b.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID))))
|
||||
sb.SelectMore(LogsV2AttributesStringColumn)
|
||||
sb.SelectMore(LogsV2AttributesNumberColumn)
|
||||
sb.SelectMore(LogsV2AttributesBoolColumn)
|
||||
sb.SelectMore(LogsV2ResourcesStringColumn)
|
||||
sb.SelectMore(LogsV2ScopeStringColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2AttributesStringColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2AttributesNumberColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2AttributesBoolColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2ResourcesStringColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2ScopeStringColumn)
|
||||
|
||||
} else {
|
||||
// Select specified columns
|
||||
for index := range query.SelectFields {
|
||||
if query.SelectFields[index].Name == LogsV2TimestampColumn || query.SelectFields[index].Name == LogsV2IDColumn {
|
||||
if query.SelectFields[index].Name == logstelemetryschema.LogsV2TimestampColumn || query.SelectFields[index].Name == logstelemetryschema.LogsV2IDColumn {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -318,7 +352,7 @@ func (b *logQueryStatementBuilder) buildListQuery(
|
||||
}
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", logstelemetryschema.DBName, logstelemetryschema.LogsV2TableName))
|
||||
// Add filter conditions
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables, skipResourceFilter)
|
||||
|
||||
@@ -424,7 +458,7 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
|
||||
}
|
||||
|
||||
// Add FROM clause
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", logstelemetryschema.DBName, logstelemetryschema.LogsV2TableName))
|
||||
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables, skipResourceFilter)
|
||||
|
||||
@@ -592,7 +626,7 @@ func (b *logQueryStatementBuilder) buildScalarQuery(
|
||||
}
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", logstelemetryschema.DBName, logstelemetryschema.LogsV2TableName))
|
||||
|
||||
// Add filter conditions
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables, skipResourceFilter)
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrylogs
|
||||
package logsstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -212,12 +213,12 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
fl := flaggertest.New(t)
|
||||
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
keysMap := buildCompleteFieldKeyMap(releaseTime)
|
||||
keysMap := logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
mockMetadataStore.KeysMap = keysMap
|
||||
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
@@ -227,7 +228,7 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -355,12 +356,12 @@ func TestStatementBuilderListQuery(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fl := flaggertest.New(t)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
fm := NewFieldMapper(fl)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
|
||||
// Create a test release time
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
@@ -370,7 +371,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -507,11 +508,11 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fl := flaggertest.New(t)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
fm := NewFieldMapper(fl)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
// Create a test release time
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
@@ -521,7 +522,7 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -585,11 +586,11 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fl := flaggertest.New(t)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
fm := NewFieldMapper(fl)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
// Create a test release time
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
@@ -599,7 +600,7 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -684,9 +685,9 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
fl := flaggertest.New(t)
|
||||
fm := NewFieldMapper(fl)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMapCollision()
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMapCollision()
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
@@ -696,7 +697,7 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -740,8 +741,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: IntrinsicFields["severity_text"],
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: logstelemetryschema.IntrinsicFields["severity_text"],
|
||||
},
|
||||
{
|
||||
name: "intrinsic field with other key match - no override",
|
||||
@@ -777,7 +778,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "severity_number",
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
@@ -791,8 +792,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["service.name"][0],
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["service.name"][0],
|
||||
},
|
||||
{
|
||||
name: "single matching key with incorrect context specified - no override",
|
||||
@@ -801,7 +802,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
@@ -815,8 +816,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["service.name"][0],
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["service.name"][0],
|
||||
},
|
||||
{
|
||||
name: "multiple matching keys - all materialized",
|
||||
@@ -825,7 +826,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "multi.mat.key",
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
@@ -839,7 +840,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "mixed.materialization.key",
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
@@ -853,8 +854,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["mixed.materialization.key"][0],
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["mixed.materialization.key"][0],
|
||||
},
|
||||
{
|
||||
name: "no matching keys - unknown field",
|
||||
@@ -863,7 +864,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "unknown.field",
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
@@ -878,7 +879,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "unknown.field",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
@@ -893,8 +894,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["mat.key"][0],
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["mat.key"][0],
|
||||
},
|
||||
{
|
||||
name: "non-materialized field",
|
||||
@@ -903,16 +904,16 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["user.id"][0],
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["user.id"][0],
|
||||
},
|
||||
}
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := NewFieldMapper(fl)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMapCollision()
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMapCollision()
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
@@ -922,7 +923,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -1054,11 +1055,11 @@ func TestStmtBuilderBodyField(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
fl := flaggertest.WithUseJSONBody(t, c.enableUseJSONBody)
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
// build the key map
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
for _, field := range IntrinsicFields {
|
||||
for _, field := range logstelemetryschema.IntrinsicFields {
|
||||
f := field
|
||||
mockMetadataStore.KeysMap[field.Name] = append(mockMetadataStore.KeysMap[field.Name], &f)
|
||||
}
|
||||
@@ -1069,7 +1070,7 @@ func TestStmtBuilderBodyField(t *testing.T) {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -1155,11 +1156,11 @@ func TestStmtBuilderBodyFullTextSearch(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
fl := flaggertest.WithUseJSONBody(t, c.enableUseJSONBody)
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
// build the key map
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
for _, field := range IntrinsicFields {
|
||||
for _, field := range logstelemetryschema.IntrinsicFields {
|
||||
f := field
|
||||
mockMetadataStore.KeysMap[field.Name] = append(mockMetadataStore.KeysMap[field.Name], &f)
|
||||
}
|
||||
@@ -1170,7 +1171,7 @@ func TestStmtBuilderBodyFullTextSearch(t *testing.T) {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -1273,14 +1274,14 @@ func newSkipResourceFingerprintLogsBuilder(
|
||||
t.Helper()
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC))
|
||||
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMap(time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC))
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(
|
||||
instrumentationtest.New().ToProviderSettings(),
|
||||
DefaultFullTextColumn,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
fm,
|
||||
cb,
|
||||
fl,
|
||||
@@ -1292,7 +1293,7 @@ func newSkipResourceFingerprintLogsBuilder(
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
fl,
|
||||
telemetryStore,
|
||||
skipEnable,
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrymeter
|
||||
package meterstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -7,8 +7,12 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -21,19 +25,40 @@ type meterQueryStatementBuilder struct {
|
||||
metadataStore telemetrytypes.MetadataStore
|
||||
fm qbtypes.FieldMapper
|
||||
cb qbtypes.ConditionBuilder
|
||||
metricsStatementBuilder *telemetrymetrics.MetricQueryStatementBuilder
|
||||
metricsStatementBuilder *metricsstatementbuilder.StatementBuilder
|
||||
}
|
||||
|
||||
var _ qbtypes.StatementBuilder[qbtypes.MetricAggregation] = (*meterQueryStatementBuilder)(nil)
|
||||
|
||||
// NewFactory returns a provider factory for the meter statement builder. Its New
|
||||
// reuses the metrics FieldMapper/ConditionBuilder and delegates the final SELECT
|
||||
// to a metrics statement builder built via the metrics factory.
|
||||
func NewFactory(
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
fl flagger.Flagger,
|
||||
) factory.ProviderFactory[qbtypes.StatementBuilder[qbtypes.MetricAggregation], statementbuilder.Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("meter"),
|
||||
func(ctx context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.StatementBuilder[qbtypes.MetricAggregation], error) {
|
||||
metricsStatementBuilder, err := metricsstatementbuilder.NewFactory(metadataStore, fl).New(ctx, settings, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fm := metricstelemetryschema.NewFieldMapper()
|
||||
cb := metricstelemetryschema.NewConditionBuilder(fm)
|
||||
return NewMeterQueryStatementBuilder(settings, metadataStore, fm, cb, metricsStatementBuilder), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func NewMeterQueryStatementBuilder(
|
||||
settings factory.ProviderSettings,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
fieldMapper qbtypes.FieldMapper,
|
||||
conditionBuilder qbtypes.ConditionBuilder,
|
||||
metricsStatementBuilder *telemetrymetrics.MetricQueryStatementBuilder,
|
||||
metricsStatementBuilder *metricsstatementbuilder.StatementBuilder,
|
||||
) *meterQueryStatementBuilder {
|
||||
metricsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrymeter")
|
||||
metricsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema")
|
||||
|
||||
return &meterQueryStatementBuilder{
|
||||
logger: metricsSettings.Logger(),
|
||||
@@ -53,7 +78,7 @@ func (b *meterQueryStatementBuilder) Build(
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
variables map[string]qbtypes.VariableItem,
|
||||
) (*qbtypes.Statement, error) {
|
||||
keySelectors := telemetrymetrics.GetKeySelectors(query)
|
||||
keySelectors := metricsstatementbuilder.GetKeySelectors(query)
|
||||
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, keySelectors)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -133,8 +158,8 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
sb.SelectMore(col)
|
||||
}
|
||||
|
||||
tbl := WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
aggCol, err := AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
tbl := metertelemetryschema.WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
aggCol, err := metertelemetryschema.AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -143,7 +168,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
}
|
||||
|
||||
sb.SelectMore(fmt.Sprintf("%s AS value", aggCol))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", DBName, tbl))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", metertelemetryschema.DBName, tbl))
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
sb.GTE("unix_milli", start),
|
||||
@@ -222,8 +247,8 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDelta(
|
||||
sb.SelectMore(col)
|
||||
}
|
||||
|
||||
tbl := WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
aggCol, err := AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality,
|
||||
tbl := metertelemetryschema.WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
aggCol, err := metertelemetryschema.AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality,
|
||||
query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
@@ -234,7 +259,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDelta(
|
||||
|
||||
sb.SelectMore(fmt.Sprintf("%s AS per_series_value", aggCol))
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", DBName, tbl))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", metertelemetryschema.DBName, tbl))
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
sb.GTE("unix_milli", start),
|
||||
@@ -300,14 +325,14 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
baseSb.SelectMore(col)
|
||||
}
|
||||
|
||||
tbl := WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
aggCol, err := AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
tbl := metertelemetryschema.WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
aggCol, err := metertelemetryschema.AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
baseSb.SelectMore(fmt.Sprintf("%s AS per_series_value", aggCol))
|
||||
|
||||
baseSb.From(fmt.Sprintf("%s.%s AS points", DBName, tbl))
|
||||
baseSb.From(fmt.Sprintf("%s.%s AS points", metertelemetryschema.DBName, tbl))
|
||||
baseSb.Where(
|
||||
baseSb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
baseSb.GTE("unix_milli", start),
|
||||
@@ -350,7 +375,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
for _, g := range query.GroupBy {
|
||||
wrapped.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
wrapped.SelectMore(fmt.Sprintf("%s AS per_series_value", telemetrymetrics.RateTmpl))
|
||||
wrapped.SelectMore(fmt.Sprintf("%s AS per_series_value", metricsstatementbuilder.RateTmpl))
|
||||
wrapped.From(fmt.Sprintf("(%s) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)", innerQuery))
|
||||
q, args := wrapped.BuildWithFlavor(sqlbuilder.ClickHouse, innerArgs...)
|
||||
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, nil
|
||||
@@ -361,7 +386,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
for _, g := range query.GroupBy {
|
||||
wrapped.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
wrapped.SelectMore(fmt.Sprintf("%s AS per_series_value", telemetrymetrics.IncreaseTmpl))
|
||||
wrapped.SelectMore(fmt.Sprintf("%s AS per_series_value", metricsstatementbuilder.IncreaseTmpl))
|
||||
wrapped.From(fmt.Sprintf("(%s) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)", innerQuery))
|
||||
q, args := wrapped.BuildWithFlavor(sqlbuilder.ClickHouse, innerArgs...)
|
||||
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, nil
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrymeter
|
||||
package meterstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -7,7 +7,8 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -158,8 +159,8 @@ func TestStatementBuilder(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
fm := telemetrymetrics.NewFieldMapper()
|
||||
cb := telemetrymetrics.NewConditionBuilder(fm)
|
||||
fm := metricstelemetryschema.NewFieldMapper()
|
||||
cb := metricstelemetryschema.NewConditionBuilder(fm)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
keys, err := telemetrytypestest.LoadFieldKeysFromJSON("testdata/keys_map.json")
|
||||
if err != nil {
|
||||
@@ -169,7 +170,7 @@ func TestStatementBuilder(t *testing.T) {
|
||||
|
||||
flagger := flaggertest.New(t)
|
||||
|
||||
metricStmtBuilder := telemetrymetrics.NewMetricQueryStatementBuilder(instrumentationtest.New().ToProviderSettings(), mockMetadataStore, fm, cb, flagger)
|
||||
metricStmtBuilder := metricsstatementbuilder.NewMetricQueryStatementBuilder(instrumentationtest.New().ToProviderSettings(), mockMetadataStore, fm, cb, flagger)
|
||||
|
||||
statementBuilder := NewMeterQueryStatementBuilder(
|
||||
instrumentationtest.New().ToProviderSettings(),
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrymetrics
|
||||
package metricsstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -118,8 +119,8 @@ func TestReducedStatementBuilder(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fm := metricstelemetryschema.NewFieldMapper()
|
||||
cb := metricstelemetryschema.NewConditionBuilder(fm)
|
||||
fl, err := flagger.New(context.Background(), instrumentationtest.New().ToProviderSettings(), flagger.Config{}, flagger.MustNewRegistry())
|
||||
require.NoError(t, err)
|
||||
sb := NewMetricQueryStatementBuilder(instrumentationtest.New().ToProviderSettings(), telemetrytypestest.NewMockMetadataStore(), fm, cb, fl)
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrymetrics
|
||||
package metricsstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -29,7 +31,7 @@ const (
|
||||
OthersMultiTemporality = `IF(LOWER(temporality) LIKE LOWER('delta'), %s, %s) AS per_series_value`
|
||||
)
|
||||
|
||||
type MetricQueryStatementBuilder struct {
|
||||
type StatementBuilder struct {
|
||||
logger *slog.Logger
|
||||
metadataStore telemetrytypes.MetadataStore
|
||||
fm qbtypes.FieldMapper
|
||||
@@ -37,7 +39,24 @@ type MetricQueryStatementBuilder struct {
|
||||
flagger flagger.Flagger
|
||||
}
|
||||
|
||||
var _ qbtypes.StatementBuilder[qbtypes.MetricAggregation] = (*MetricQueryStatementBuilder)(nil)
|
||||
var _ qbtypes.StatementBuilder[qbtypes.MetricAggregation] = (*StatementBuilder)(nil)
|
||||
|
||||
// NewFactory returns a provider factory for the metrics statement builder. Its
|
||||
// New internalizes the FieldMapper and ConditionBuilder and yields the concrete
|
||||
// *StatementBuilder so the meter builder can reuse it.
|
||||
func NewFactory(
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
fl flagger.Flagger,
|
||||
) factory.ProviderFactory[*StatementBuilder, statementbuilder.Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("metrics"),
|
||||
func(_ context.Context, settings factory.ProviderSettings, _ statementbuilder.Config) (*StatementBuilder, error) {
|
||||
fm := metricstelemetryschema.NewFieldMapper()
|
||||
cb := metricstelemetryschema.NewConditionBuilder(fm)
|
||||
return NewMetricQueryStatementBuilder(settings, metadataStore, fm, cb, fl), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func NewMetricQueryStatementBuilder(
|
||||
settings factory.ProviderSettings,
|
||||
@@ -45,9 +64,9 @@ func NewMetricQueryStatementBuilder(
|
||||
fieldMapper qbtypes.FieldMapper,
|
||||
conditionBuilder qbtypes.ConditionBuilder,
|
||||
flagger flagger.Flagger,
|
||||
) *MetricQueryStatementBuilder {
|
||||
metricsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrymetrics")
|
||||
return &MetricQueryStatementBuilder{
|
||||
) *StatementBuilder {
|
||||
metricsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema")
|
||||
return &StatementBuilder{
|
||||
logger: metricsSettings.Logger(),
|
||||
metadataStore: metadataStore,
|
||||
fm: fieldMapper,
|
||||
@@ -89,7 +108,7 @@ func GetKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation])
|
||||
return keySelectors
|
||||
}
|
||||
|
||||
func (b *MetricQueryStatementBuilder) Build(
|
||||
func (b *StatementBuilder) Build(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
start uint64,
|
||||
@@ -126,7 +145,7 @@ func (b *MetricQueryStatementBuilder) Build(
|
||||
return b.buildPipelineStatement(ctx, orgID, start, end, query, keys, variables)
|
||||
}
|
||||
|
||||
func (b *MetricQueryStatementBuilder) buildPipelineStatement(
|
||||
func (b *StatementBuilder) buildPipelineStatement(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
start, end uint64,
|
||||
@@ -188,11 +207,11 @@ func (b *MetricQueryStatementBuilder) buildPipelineStatement(
|
||||
// shaped exactly like samples_v4 / time_series_v4, so once the table names are
|
||||
// chosen the rest of the pipeline is unchanged.
|
||||
useBuffer := agg.Reduced &&
|
||||
end-start < oneDayInMilliseconds &&
|
||||
start >= uint64(time.Now().UnixMilli())-oneDayInMilliseconds
|
||||
end-start < metricstelemetryschema.OneDayInMilliseconds &&
|
||||
start >= uint64(time.Now().UnixMilli())-metricstelemetryschema.OneDayInMilliseconds
|
||||
|
||||
samplesTable, _ := WhichSamplesTableToUse(start, end, agg.Type, agg.TimeAggregation, useBuffer, agg.TableHints)
|
||||
tsStart, tsEnd, _, tsTable := WhichTSTableToUse(start, end, useBuffer, agg.TableHints)
|
||||
samplesTable, _ := metricstelemetryschema.WhichSamplesTableToUse(start, end, agg.Type, agg.TimeAggregation, useBuffer, agg.TableHints)
|
||||
tsStart, tsEnd, _, tsTable := metricstelemetryschema.WhichTSTableToUse(start, end, useBuffer, agg.TableHints)
|
||||
|
||||
var timeSeriesCTE string
|
||||
var timeSeriesCTEArgs []any
|
||||
@@ -232,7 +251,7 @@ func (b *MetricQueryStatementBuilder) buildPipelineStatement(
|
||||
var tsCTE string
|
||||
var tsArgs []any
|
||||
// time series rows are written on hour boundaries
|
||||
tsStart := start - (start % oneHourInMilliseconds)
|
||||
tsStart := start - (start % metricstelemetryschema.OneHourInMilliseconds)
|
||||
if tsCTE, tsArgs, err = b.buildReducedTimeSeriesCTE(ctx, orgID, tsStart, end, query, keys, variables); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -282,7 +301,7 @@ func unionStatements(main, reduced *qbtypes.Statement, query qbtypes.QueryBuilde
|
||||
return &qbtypes.Statement{Query: q, Args: args, Warnings: warnings}, nil
|
||||
}
|
||||
|
||||
func (b *MetricQueryStatementBuilder) buildReducedTimeSeriesCTE(
|
||||
func (b *StatementBuilder) buildReducedTimeSeriesCTE(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
start, end uint64,
|
||||
@@ -312,7 +331,7 @@ func (b *MetricQueryStatementBuilder) buildReducedTimeSeriesCTE(
|
||||
}
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, TimeseriesV4ReducedLocalTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedLocalTableName))
|
||||
sb.Select("fingerprint")
|
||||
for _, g := range query.GroupBy {
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys)
|
||||
@@ -343,7 +362,7 @@ func (b *MetricQueryStatementBuilder) buildReducedTimeSeriesCTE(
|
||||
// spatial_aggregation_cte in one level with no per-series grouping, so shards
|
||||
// send one state per (step, group) instead of per (series, step, group).
|
||||
// FINAL still dedups recomputed 60s buckets at scan time.
|
||||
func (b *MetricQueryStatementBuilder) buildReducedSpatialAggFastPath(
|
||||
func (b *StatementBuilder) buildReducedSpatialAggFastPath(
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
timeSeriesCTE string,
|
||||
@@ -352,7 +371,7 @@ func (b *MetricQueryStatementBuilder) buildReducedSpatialAggFastPath(
|
||||
agg := query.Aggregations[0]
|
||||
stepSec := int64(query.StepInterval.Seconds())
|
||||
|
||||
value, _, ok := ReducedValueColumn(agg.Type, agg.SpaceAggregation)
|
||||
value, _, ok := metricstelemetryschema.ReducedValueColumn(agg.Type, agg.SpaceAggregation)
|
||||
if !ok {
|
||||
return "", nil, false
|
||||
}
|
||||
@@ -362,8 +381,8 @@ func (b *MetricQueryStatementBuilder) buildReducedSpatialAggFastPath(
|
||||
for _, g := range query.GroupBy {
|
||||
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
sb.SelectMore(fmt.Sprintf("%s AS value", ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec, value)))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points FINAL", DBName, WhichReducedSamplesTableToUse(agg.Type)))
|
||||
sb.SelectMore(fmt.Sprintf("%s AS value", metricstelemetryschema.ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec, value)))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points FINAL", metricstelemetryschema.DBName, metricstelemetryschema.WhichReducedSamplesTableToUse(agg.Type)))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.reduced_fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", agg.MetricName),
|
||||
@@ -377,7 +396,7 @@ func (b *MetricQueryStatementBuilder) buildReducedSpatialAggFastPath(
|
||||
return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args, true
|
||||
}
|
||||
|
||||
func (b *MetricQueryStatementBuilder) buildReducedTemporalAggregationCTE(
|
||||
func (b *StatementBuilder) buildReducedTemporalAggregationCTE(
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
timeSeriesCTE string,
|
||||
@@ -386,7 +405,7 @@ func (b *MetricQueryStatementBuilder) buildReducedTemporalAggregationCTE(
|
||||
agg := query.Aggregations[0]
|
||||
stepSec := int64(query.StepInterval.Seconds())
|
||||
|
||||
value, weight, ok := ReducedValueColumn(agg.Type, agg.SpaceAggregation)
|
||||
value, weight, ok := metricstelemetryschema.ReducedValueColumn(agg.Type, agg.SpaceAggregation)
|
||||
if !ok {
|
||||
return "", nil, false
|
||||
}
|
||||
@@ -399,13 +418,13 @@ func (b *MetricQueryStatementBuilder) buildReducedTemporalAggregationCTE(
|
||||
for _, g := range query.GroupBy {
|
||||
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
sb.SelectMore(fmt.Sprintf("%s AS per_series_value", ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec, value)))
|
||||
sb.SelectMore(fmt.Sprintf("%s AS per_series_value", metricstelemetryschema.ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec, value)))
|
||||
if weight != "" {
|
||||
// count_series is a series count, not additive over time, so the avg
|
||||
// denominator is reduced with avg
|
||||
sb.SelectMore(fmt.Sprintf("avg(%s) AS per_series_weight", weight))
|
||||
}
|
||||
sb.From(fmt.Sprintf("%s.%s AS points FINAL", DBName, WhichReducedSamplesTableToUse(agg.Type)))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points FINAL", metricstelemetryschema.DBName, metricstelemetryschema.WhichReducedSamplesTableToUse(agg.Type)))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.reduced_fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", agg.MetricName),
|
||||
@@ -419,7 +438,7 @@ func (b *MetricQueryStatementBuilder) buildReducedTemporalAggregationCTE(
|
||||
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, true
|
||||
}
|
||||
|
||||
func (b *MetricQueryStatementBuilder) buildReducedSpatialAggregationCTE(
|
||||
func (b *StatementBuilder) buildReducedSpatialAggregationCTE(
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
) (string, []any) {
|
||||
spatial := "sum(per_series_value)"
|
||||
@@ -446,7 +465,7 @@ func (b *MetricQueryStatementBuilder) buildReducedSpatialAggregationCTE(
|
||||
return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args
|
||||
}
|
||||
|
||||
func (b *MetricQueryStatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
func (b *StatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
samplesTable string,
|
||||
@@ -465,7 +484,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
|
||||
aggCol, err := AggregationColumnForSamplesTable(
|
||||
aggCol, err := metricstelemetryschema.AggregationColumnForSamplesTable(
|
||||
samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation,
|
||||
)
|
||||
if err != nil {
|
||||
@@ -483,7 +502,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
|
||||
sb.SelectMore(fmt.Sprintf("%s AS value", aggCol))
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", DBName, samplesTable))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
@@ -497,7 +516,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args, nil
|
||||
}
|
||||
|
||||
func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE(
|
||||
func (b *StatementBuilder) buildTimeSeriesCTE(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
start, end uint64,
|
||||
@@ -529,7 +548,7 @@ func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE(
|
||||
}
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, tsTable))
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, tsTable))
|
||||
|
||||
sb.Select("fingerprint")
|
||||
for _, g := range query.GroupBy {
|
||||
@@ -552,7 +571,7 @@ func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE(
|
||||
|
||||
// the buffer holds both raw rows and the reduced catalog rows; the raw read
|
||||
// only wants the original series
|
||||
if tsTable == TimeseriesV4BufferLocalTableName {
|
||||
if tsTable == metricstelemetryschema.TimeseriesV4BufferLocalTableName {
|
||||
sb.Where(sb.EQ("is_reduced", false))
|
||||
}
|
||||
|
||||
@@ -567,7 +586,7 @@ func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE(
|
||||
return fmt.Sprintf("(%s) AS filtered_time_series", q), args, nil
|
||||
}
|
||||
|
||||
func (b *MetricQueryStatementBuilder) buildTemporalAggregationCTE(
|
||||
func (b *StatementBuilder) buildTemporalAggregationCTE(
|
||||
ctx context.Context,
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
@@ -584,7 +603,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggregationCTE(
|
||||
return b.buildTemporalAggForMultipleTemporalities(ctx, start, end, query, samplesTable, timeSeriesCTE, timeSeriesCTEArgs)
|
||||
}
|
||||
|
||||
func (b *MetricQueryStatementBuilder) buildTemporalAggDelta(
|
||||
func (b *StatementBuilder) buildTemporalAggDelta(
|
||||
_ context.Context,
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
@@ -605,7 +624,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggDelta(
|
||||
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
|
||||
aggCol, err := AggregationColumnForSamplesTable(samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation)
|
||||
aggCol, err := metricstelemetryschema.AggregationColumnForSamplesTable(samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -616,7 +635,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggDelta(
|
||||
|
||||
sb.SelectMore(fmt.Sprintf("%s AS per_series_value", aggCol))
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", DBName, samplesTable))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
@@ -631,7 +650,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggDelta(
|
||||
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, nil
|
||||
}
|
||||
|
||||
func (b *MetricQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
func (b *StatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
_ context.Context,
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
@@ -651,13 +670,13 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
baseSb.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
|
||||
aggCol, err := AggregationColumnForSamplesTable(samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation)
|
||||
aggCol, err := metricstelemetryschema.AggregationColumnForSamplesTable(samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
baseSb.SelectMore(fmt.Sprintf("%s AS per_series_value", aggCol))
|
||||
|
||||
baseSb.From(fmt.Sprintf("%s.%s AS points", DBName, samplesTable))
|
||||
baseSb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
|
||||
baseSb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
baseSb.Where(
|
||||
baseSb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
@@ -697,7 +716,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
}
|
||||
}
|
||||
|
||||
func (b *MetricQueryStatementBuilder) buildTemporalAggForMultipleTemporalities(
|
||||
func (b *StatementBuilder) buildTemporalAggForMultipleTemporalities(
|
||||
_ context.Context,
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
@@ -716,11 +735,11 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggForMultipleTemporalities(
|
||||
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
|
||||
aggForDeltaTemporality, err := AggregationColumnForSamplesTable(samplesTable, metrictypes.Delta, query.Aggregations[0].TimeAggregation)
|
||||
aggForDeltaTemporality, err := metricstelemetryschema.AggregationColumnForSamplesTable(samplesTable, metrictypes.Delta, query.Aggregations[0].TimeAggregation)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
aggForCumulativeTemporality, err := AggregationColumnForSamplesTable(samplesTable, metrictypes.Cumulative, query.Aggregations[0].TimeAggregation)
|
||||
aggForCumulativeTemporality, err := metricstelemetryschema.AggregationColumnForSamplesTable(samplesTable, metrictypes.Cumulative, query.Aggregations[0].TimeAggregation)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -748,7 +767,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggForMultipleTemporalities(
|
||||
sb.SelectMore(expr)
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", DBName, samplesTable))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
@@ -762,7 +781,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggForMultipleTemporalities(
|
||||
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", queryWithWindowAndOrder), args, nil
|
||||
}
|
||||
|
||||
func (b *MetricQueryStatementBuilder) buildSpatialAggregationCTE(
|
||||
func (b *StatementBuilder) buildSpatialAggregationCTE(
|
||||
_ context.Context,
|
||||
_ uint64,
|
||||
_ uint64,
|
||||
@@ -788,7 +807,7 @@ func (b *MetricQueryStatementBuilder) buildSpatialAggregationCTE(
|
||||
return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args
|
||||
}
|
||||
|
||||
func (b *MetricQueryStatementBuilder) BuildFinalSelect(
|
||||
func (b *StatementBuilder) BuildFinalSelect(
|
||||
cteFragments []string,
|
||||
cteArgs [][]any,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
@@ -833,7 +852,7 @@ func (b *MetricQueryStatementBuilder) BuildFinalSelect(
|
||||
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
|
||||
aggQuery, err := AggregationQueryForHistogramCountWithParams(query.Aggregations[0].ComparisonSpaceAggregationParam)
|
||||
aggQuery, err := metricstelemetryschema.AggregationQueryForHistogramCountWithParams(query.Aggregations[0].ComparisonSpaceAggregationParam)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrymetrics
|
||||
package metricsstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -252,8 +253,8 @@ func TestStatementBuilder(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fm := metricstelemetryschema.NewFieldMapper()
|
||||
cb := metricstelemetryschema.NewConditionBuilder(fm)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
keys, err := telemetrytypestest.LoadFieldKeysFromJSON("testdata/keys_map.json")
|
||||
if err != nil {
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetryresourcefilter
|
||||
package resourcefilter
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetryresourcefilter
|
||||
package resourcefilter
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetryresourcefilter
|
||||
package resourcefilter
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetryresourcefilter
|
||||
package resourcefilter
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetryresourcefilter
|
||||
package resourcefilter
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -45,7 +45,7 @@ func New[T any](
|
||||
fullTextColumn *telemetrytypes.TelemetryFieldKey,
|
||||
fl flagger.Flagger,
|
||||
) *resourceFilterStatementBuilder[T] {
|
||||
set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryresourcefilter")
|
||||
set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/statementbuilder/resourcefilter")
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
return &resourceFilterStatementBuilder[T]{
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetryresourcefilter
|
||||
package resourcefilter
|
||||
|
||||
import (
|
||||
"context"
|
||||
208
pkg/statementbuilder/scopedtracesstatementbuilder/aggregate.go
Normal file
208
pkg/statementbuilder/scopedtracesstatementbuilder/aggregate.go
Normal file
@@ -0,0 +1,208 @@
|
||||
package scopedtracesstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// This file holds the Aggregate constructors a TraceScope's columns are declared
|
||||
// with. All SQL rendering goes through the fieldMapper.
|
||||
|
||||
// Aggregate renders one column's SQL through the fieldMapper and lists the attribute
|
||||
// keys it references so the builder can pre-fetch their metadata. Build one with the
|
||||
// constructors below; the zero value is not usable.
|
||||
type Aggregate struct {
|
||||
keys []*telemetrytypes.TelemetryFieldKey
|
||||
render func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (expr string, args []any, err error)
|
||||
}
|
||||
|
||||
// IntrinsicSpanKey references an intrinsic span-index field (timestamp, name, …) by
|
||||
// its canonical name; the field mapper resolves it to the physical column.
|
||||
func IntrinsicSpanKey(name string) *telemetrytypes.TelemetryFieldKey {
|
||||
return &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextSpan,
|
||||
}
|
||||
}
|
||||
|
||||
// AggFunc is a ClickHouse aggregate function name.
|
||||
type AggFunc string
|
||||
|
||||
const (
|
||||
AggSum AggFunc = "sum"
|
||||
AggMax AggFunc = "max"
|
||||
AggMin AggFunc = "min"
|
||||
)
|
||||
|
||||
// PickDirection selects the earliest (argMin) or latest (argMax) span by ordering.
|
||||
type PickDirection int
|
||||
|
||||
const (
|
||||
PickLatest PickDirection = iota
|
||||
PickEarliest
|
||||
)
|
||||
|
||||
// CountAll renders count().
|
||||
func CountAll() Aggregate {
|
||||
return Aggregate{render: func(context.Context, valuer.UUID, uint64, uint64, *fieldMapper) (string, []any, error) {
|
||||
return "count()", nil, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// FieldReduce renders <fn>(<field>) over a field-mapper-resolved column.
|
||||
func FieldReduce(fn AggFunc, key *telemetrytypes.TelemetryFieldKey) Aggregate {
|
||||
return Aggregate{render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
|
||||
f, err := m.FieldFor(ctx, orgID, startNs, endNs, key)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return fmt.Sprintf("%s(%s)", fn, f), nil, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// TraceDuration renders the full-trace wall duration: last span end minus first
|
||||
// span start.
|
||||
func TraceDuration(tsKey, durationKey *telemetrytypes.TelemetryFieldKey) Aggregate {
|
||||
return Aggregate{render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
|
||||
ts, err := m.FieldFor(ctx, orgID, startNs, endNs, tsKey)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
dur, err := m.FieldFor(ctx, orgID, startNs, endNs, durationKey)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return fmt.Sprintf("(max(toUnixTimestamp64Nano(%s) + %s) - min(toUnixTimestamp64Nano(%s)))", ts, dur, ts), nil, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// FieldAnyWhere renders anyIf(<field>, <cond>) — the field value from any span
|
||||
// matching the condition.
|
||||
func FieldAnyWhere(valueKey, condKey *telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, condValue any) Aggregate {
|
||||
return Aggregate{render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
|
||||
v, err := m.FieldFor(ctx, orgID, startNs, endNs, valueKey)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
cond, args, err := m.ConditionFor(ctx, orgID, startNs, endNs, condKey, op, condValue)
|
||||
return fmt.Sprintf("anyIf(%s, %s)", v, cond), args, err
|
||||
}}
|
||||
}
|
||||
|
||||
// AnyValue renders any(<value>) over a metadata-resolved attribute value.
|
||||
func AnyValue(key *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType) Aggregate {
|
||||
return Aggregate{keys: keysOf(key), render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
|
||||
v, args, err := m.ValueFor(ctx, orgID, startNs, endNs, key, dt)
|
||||
return fmt.Sprintf("any(%s)", v), args, err
|
||||
}}
|
||||
}
|
||||
|
||||
// CountExists renders countIf(<key> EXISTS) — counts spans carrying key.
|
||||
func CountExists(key *telemetrytypes.TelemetryFieldKey) Aggregate {
|
||||
return Aggregate{keys: keysOf(key), render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
|
||||
cond, args, err := m.ExistsFor(ctx, orgID, startNs, endNs, key)
|
||||
return fmt.Sprintf("countIf(%s)", cond), args, err
|
||||
}}
|
||||
}
|
||||
|
||||
// CondCount renders countIf(<cond>) over a condition-builder-resolved predicate.
|
||||
func CondCount(key *telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, value any) Aggregate {
|
||||
return Aggregate{render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
|
||||
cond, args, err := m.ConditionFor(ctx, orgID, startNs, endNs, key, op, value)
|
||||
return fmt.Sprintf("countIf(%s)", cond), args, err
|
||||
}}
|
||||
}
|
||||
|
||||
// Reduce renders <fn>(<value>) over a resolved numeric attribute value.
|
||||
func Reduce(fn AggFunc, valueKey *telemetrytypes.TelemetryFieldKey) Aggregate {
|
||||
return Aggregate{keys: keysOf(valueKey), render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
|
||||
v, args, err := m.ValueFor(ctx, orgID, startNs, endNs, valueKey, telemetrytypes.FieldDataTypeFloat64)
|
||||
return fmt.Sprintf("%s(%s)", fn, v), args, err
|
||||
}}
|
||||
}
|
||||
|
||||
// ScopedReduce renders <fn>If(<field>, <gate mask>) over a field-mapper-resolved column.
|
||||
func ScopedReduce(fn AggFunc, key *telemetrytypes.TelemetryFieldKey) Aggregate {
|
||||
return Aggregate{render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
|
||||
f, err := m.FieldFor(ctx, orgID, startNs, endNs, key)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return fmt.Sprintf("%sIf(%s, %s)", fn, f, m.maskExpr), append([]any{}, m.maskArgs...), nil
|
||||
}}
|
||||
}
|
||||
|
||||
// ScopedToKeyColumn renders <fn>If(<field>, <scopeKey> EXISTS) — a span-index field
|
||||
// aggregated over spans carrying scopeKey (e.g. max LLM latency).
|
||||
func ScopedToKeyColumn(fn AggFunc, columnKey, scopeKey *telemetrytypes.TelemetryFieldKey) Aggregate {
|
||||
return Aggregate{keys: keysOf(scopeKey), render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
|
||||
col, err := m.FieldFor(ctx, orgID, startNs, endNs, columnKey)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
cond, args, err := m.ExistsFor(ctx, orgID, startNs, endNs, scopeKey)
|
||||
return fmt.Sprintf("%sIf(%s, %s)", fn, col, cond), args, err
|
||||
}}
|
||||
}
|
||||
|
||||
// PickBy renders argMinIf/argMaxIf(<value>, <orderField>, <value> EXISTS) — the value
|
||||
// from the earliest/latest span that carries it.
|
||||
func PickBy(valueKey *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType, orderKey *telemetrytypes.TelemetryFieldKey, dir PickDirection) Aggregate {
|
||||
fn := "argMaxIf"
|
||||
if dir == PickEarliest {
|
||||
fn = "argMinIf"
|
||||
}
|
||||
return Aggregate{keys: keysOf(valueKey), render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
|
||||
v, vargs, err := m.ValueFor(ctx, orgID, startNs, endNs, valueKey, dt)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
order, err := m.FieldFor(ctx, orgID, startNs, endNs, orderKey)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
cond, cargs, err := m.ExistsFor(ctx, orgID, startNs, endNs, valueKey)
|
||||
return fmt.Sprintf("%s(%s, %s, %s)", fn, v, order, cond), append(vargs, cargs...), err
|
||||
}}
|
||||
}
|
||||
|
||||
// UniqCount renders uniqIf(<value>, <value> EXISTS) — distinct count of an attribute.
|
||||
func UniqCount(valueKey *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType) Aggregate {
|
||||
return Aggregate{keys: keysOf(valueKey), render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
|
||||
v, vargs, err := m.ValueFor(ctx, orgID, startNs, endNs, valueKey, dt)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
cond, cargs, err := m.ExistsFor(ctx, orgID, startNs, endNs, valueKey)
|
||||
return fmt.Sprintf("uniqIf(%s, %s)", v, cond), append(vargs, cargs...), err
|
||||
}}
|
||||
}
|
||||
|
||||
// SumOfKeys renders coalesce(sum(<v1>), 0) + coalesce(sum(<v2>), 0) + … over several
|
||||
// numeric attributes. Coalesced because a key absent from every span sums to NULL and
|
||||
// NULL + n = NULL — a trace with only output tokens would otherwise total NULL.
|
||||
func SumOfKeys(dt telemetrytypes.FieldDataType, valueKeys ...*telemetrytypes.TelemetryFieldKey) Aggregate {
|
||||
return Aggregate{keys: valueKeys, render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
|
||||
parts := make([]string, 0, len(valueKeys))
|
||||
var args []any
|
||||
for _, k := range valueKeys {
|
||||
v, vargs, err := m.ValueFor(ctx, orgID, startNs, endNs, k, dt)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("coalesce(sum(%s), 0)", v))
|
||||
args = append(args, vargs...)
|
||||
}
|
||||
return strings.Join(parts, " + "), args, nil
|
||||
}}
|
||||
}
|
||||
|
||||
func keysOf(k *telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
|
||||
return []*telemetrytypes.TelemetryFieldKey{k}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package scopedtracesstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
)
|
||||
|
||||
// fieldMapper resolves aggregate-column SQL through the shared field mapper and
|
||||
// condition builder, following their method shapes (FieldFor / ConditionFor / …) so
|
||||
// column resolution reads like the other statement builders. keys is the fetched
|
||||
// metadata for the keys the columns reference; the gate mask is set by the builder
|
||||
// after resolveMask (Scoped* aggregates embed it). All returned expressions are
|
||||
// escaped once, ready to embed in an outer builder.
|
||||
type fieldMapper struct {
|
||||
fm qbtypes.FieldMapper
|
||||
cb qbtypes.ConditionBuilder
|
||||
keys map[string][]*telemetrytypes.TelemetryFieldKey
|
||||
maskExpr string
|
||||
maskArgs []any
|
||||
}
|
||||
|
||||
func newFieldMapper(fm qbtypes.FieldMapper, cb qbtypes.ConditionBuilder, keys map[string][]*telemetrytypes.TelemetryFieldKey) *fieldMapper {
|
||||
return &fieldMapper{fm: fm, cb: cb, keys: keys}
|
||||
}
|
||||
|
||||
// FieldFor returns the column expression for key via the field mapper.
|
||||
func (r *fieldMapper) FieldFor(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
return r.fm.FieldFor(ctx, orgID, startNs, endNs, key)
|
||||
|
||||
}
|
||||
|
||||
// ConditionFor returns a boolean predicate for key via the condition builder
|
||||
// (materialized column when present, else map access).
|
||||
func (r *fieldMapper) ConditionFor(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, value any) (string, []any, error) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
// The condition builder owns key resolution: hand it the raw key plus the full
|
||||
// metadata map and it matches/synthesizes the candidates itself.
|
||||
conds, _, err := r.cb.ConditionFor(ctx, orgID, startNs, endNs, key, r.keys, qbtypes.ConditionBuilderOptions{}, op, value, sb)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if len(conds) == 0 {
|
||||
return "", nil, nil
|
||||
}
|
||||
// One condition per candidate variant (a key can be ingested under several data
|
||||
// types); OR them all, like the visitor does for EXISTS.
|
||||
if len(conds) == 1 {
|
||||
sb.Where(conds[0])
|
||||
} else {
|
||||
sb.Where(sb.Or(conds...))
|
||||
}
|
||||
expr, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
expr = strings.TrimPrefix(expr, "WHERE ")
|
||||
return sqlbuilder.Escape(expr), args, nil
|
||||
}
|
||||
|
||||
// ExistsFor returns the EXISTS predicate for key.
|
||||
func (r *fieldMapper) ExistsFor(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (string, []any, error) {
|
||||
return r.ConditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorExists, nil)
|
||||
}
|
||||
|
||||
// ValueFor returns the value expression for an attribute key.
|
||||
func (r *fieldMapper) ValueFor(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType) (string, []any, error) {
|
||||
// The scope declares keys statically, so a key like service.name can have evolutions.
|
||||
// this is added for the evolution.
|
||||
if cands := r.keys[key.Name]; len(cands) > 0 {
|
||||
key = cands[0]
|
||||
}
|
||||
expr, err := r.fm.ColumnExpressionFor(ctx, orgID, startNs, endNs, key, dt, r.keys)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
// Escape before embedding in the outer builder: a materialized column name carries
|
||||
// `$$` (from the dotted attribute name), which go-sqlbuilder's Build would otherwise
|
||||
// unescape to a single `$` and reference the wrong column.
|
||||
return sqlbuilder.Escape(expr), nil, nil
|
||||
}
|
||||
66
pkg/statementbuilder/scopedtracesstatementbuilder/scope.go
Normal file
66
pkg/statementbuilder/scopedtracesstatementbuilder/scope.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package scopedtracesstatementbuilder
|
||||
|
||||
import (
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
// TraceScope is the configuration the scoped trace builder accepts: which spans are
|
||||
// in scope and which per-trace columns the list computes. It only declares the gate
|
||||
// and the columns; the builder resolves everything through the field mapper, so
|
||||
// attribute access stays materialization-aware. A new span category only needs a
|
||||
// new TraceScope.
|
||||
type TraceScope struct {
|
||||
// FilterExpression is the grammar-level (EXISTS) gate, used on the delegated
|
||||
// span-list path.
|
||||
FilterExpression string
|
||||
// FieldKeys are the gate's keys, used to build the per-span mask
|
||||
// (OR of resolved EXISTS conditions).
|
||||
FieldKeys []*telemetrytypes.TelemetryFieldKey
|
||||
// Columns are the per-trace output columns.
|
||||
Columns []TraceColumn
|
||||
// DefaultOrderAlias is sorted by (desc) when the query gives no order.
|
||||
DefaultOrderAlias string
|
||||
}
|
||||
|
||||
// TraceColumn is one per-trace output column.
|
||||
type TraceColumn struct {
|
||||
// Alias must not reuse a physical span-index column name (e.g. duration_nano):
|
||||
// ClickHouse resolves bare identifiers to same-SELECT aliases first, so any
|
||||
// expression referencing that column would silently bind to the alias.
|
||||
Alias string
|
||||
// Orderable columns can be used in ORDER BY and the aggregate filter. All-span
|
||||
// aggregates (span_count, trace_duration_nano, …) are display-only and set false.
|
||||
Orderable bool
|
||||
// SpanLevel columns surface a real span/resource attribute (service.name,
|
||||
// input/output messages); a filter on them is applied span-level, so they are
|
||||
// excluded from the trace-level aliases.
|
||||
SpanLevel bool
|
||||
Expr Aggregate
|
||||
}
|
||||
|
||||
// CommonTraceColumns are domain-neutral columns any trace list can reuse. All
|
||||
// aggregate over every span, so none is Orderable.
|
||||
func CommonTraceColumns() []TraceColumn {
|
||||
ts := IntrinsicSpanKey("timestamp")
|
||||
duration := IntrinsicSpanKey("duration_nano")
|
||||
name := IntrinsicSpanKey("name")
|
||||
parentSpanID := IntrinsicSpanKey("parent_span_id")
|
||||
serviceName := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
return []TraceColumn{
|
||||
{Alias: "start_time", Expr: FieldReduce(AggMin, ts)},
|
||||
{Alias: "end_time", Expr: FieldReduce(AggMax, ts)},
|
||||
// Not plain "duration_nano": that name is the intrinsic span field, and an
|
||||
// alias would shadow it — both in ClickHouse identifier resolution and in
|
||||
// bare-name filter classification.
|
||||
{Alias: "trace_duration_nano", Expr: TraceDuration(ts, duration)},
|
||||
{Alias: "span_count", Expr: CountAll()},
|
||||
{Alias: "root_span_name", Expr: FieldAnyWhere(name, parentSpanID, qbtypes.FilterOperatorEqual, "")},
|
||||
{Alias: "service.name", SpanLevel: true, Expr: AnyValue(serviceName, telemetrytypes.FieldDataTypeString)},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,802 @@
|
||||
package scopedtracesstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/resourcefilter"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/tracesstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
qbvariables "github.com/SigNoz/signoz/pkg/variables"
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrUnsupportedRequestType = errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported request type for the scoped trace builder")
|
||||
)
|
||||
|
||||
// scopedTraceStatementBuilder builds a trace list scoped to one span category
|
||||
// (e.g. gen_ai spans). The query shape is fixed; the TraceScope decides which spans
|
||||
// are in scope and which per-trace columns to compute, so a new category only needs
|
||||
// a new scope.
|
||||
type scopedTraceStatementBuilder struct {
|
||||
logger *slog.Logger
|
||||
metadataStore telemetrytypes.MetadataStore
|
||||
fm qbtypes.FieldMapper
|
||||
cb qbtypes.ConditionBuilder
|
||||
scope TraceScope
|
||||
traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
|
||||
resourceFilterStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
|
||||
}
|
||||
|
||||
var _ qbtypes.StatementBuilder[qbtypes.TraceAggregation] = (*scopedTraceStatementBuilder)(nil)
|
||||
|
||||
// NewFactory returns a provider factory for a scoped trace statement builder. Unlike
|
||||
// the per-signal factories this package is domain-neutral, so the caller supplies the
|
||||
// factory name and the TraceScope (see aistatementbuilder for the gen_ai scope). Its
|
||||
// New delegates the span-list path to a trace statement builder built via the traces
|
||||
// factory — mirroring how the meter factory builds its own metrics builder.
|
||||
func NewFactory(
|
||||
name factory.Name,
|
||||
scope TraceScope,
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
fl flagger.Flagger,
|
||||
) factory.ProviderFactory[qbtypes.StatementBuilder[qbtypes.TraceAggregation], statementbuilder.Config] {
|
||||
return factory.NewProviderFactory(
|
||||
name,
|
||||
func(ctx context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.StatementBuilder[qbtypes.TraceAggregation], error) {
|
||||
traceStmtBuilder, err := tracesstatementbuilder.NewFactory(telemetryStore, metadataStore, fl).New(ctx, settings, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewScopedTraceStatementBuilder(settings, metadataStore, scope, traceStmtBuilder, fl), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// NewScopedTraceStatementBuilder wires the generic trace-list builder. The field
|
||||
// mapper / condition builder are built here, not injected — the list always scans the
|
||||
// traces span index. traceStmtBuilder (the delegate for the span-list path) is
|
||||
// injected because NewFactory already builds the canonical instance.
|
||||
func NewScopedTraceStatementBuilder(
|
||||
settings factory.ProviderSettings,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
scope TraceScope,
|
||||
traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
|
||||
fl flagger.Flagger,
|
||||
) qbtypes.StatementBuilder[qbtypes.TraceAggregation] {
|
||||
scopedSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/statementbuilder/scopedtracesstatementbuilder")
|
||||
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
|
||||
// Same resource-fingerprint prune as the standard trace builder — the list scans
|
||||
// the same span index.
|
||||
resourceFilterStmtBuilder := resourcefilter.New[qbtypes.TraceAggregation](
|
||||
settings,
|
||||
tracestelemetryschema.DBName,
|
||||
tracestelemetryschema.TracesResourceV3TableName,
|
||||
telemetrytypes.SignalTraces,
|
||||
telemetrytypes.SourceUnspecified,
|
||||
metadataStore,
|
||||
nil,
|
||||
fl,
|
||||
)
|
||||
|
||||
return &scopedTraceStatementBuilder{
|
||||
logger: scopedSettings.Logger(),
|
||||
metadataStore: metadataStore,
|
||||
fm: fm,
|
||||
cb: cb,
|
||||
scope: scope,
|
||||
traceStmtBuilder: traceStmtBuilder,
|
||||
resourceFilterStmtBuilder: resourceFilterStmtBuilder,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *scopedTraceStatementBuilder) Build(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
start uint64,
|
||||
end uint64,
|
||||
requestType qbtypes.RequestType,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
|
||||
variables map[string]qbtypes.VariableItem,
|
||||
) (*qbtypes.Statement, error) {
|
||||
switch requestType {
|
||||
case qbtypes.RequestTypeTrace:
|
||||
return b.buildTraceListQuery(ctx, orgID, querybuilder.ToNanoSecs(start), querybuilder.ToNanoSecs(end), query, variables)
|
||||
case qbtypes.RequestTypeRaw:
|
||||
return b.buildDelegated(ctx, orgID, start, end, requestType, query, variables)
|
||||
default:
|
||||
return nil, ErrUnsupportedRequestType
|
||||
}
|
||||
}
|
||||
|
||||
// buildDelegated ANDs the base gate into the user filter and delegates to the
|
||||
// standard trace builder (the span-list / raw path).
|
||||
func (b *scopedTraceStatementBuilder) buildDelegated(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
start, end uint64,
|
||||
requestType qbtypes.RequestType,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
|
||||
variables map[string]qbtypes.VariableItem,
|
||||
) (*qbtypes.Statement, error) {
|
||||
gate := b.scope.FilterExpression
|
||||
expr := gate
|
||||
if query.Filter != nil && strings.TrimSpace(query.Filter.Expression) != "" {
|
||||
expr = fmt.Sprintf("(%s) AND (%s)", gate, query.Filter.Expression)
|
||||
}
|
||||
|
||||
// shallow copy; only Filter is replaced, caller's query untouched
|
||||
gated := query
|
||||
gated.Filter = &qbtypes.Filter{Expression: expr}
|
||||
|
||||
return b.traceStmtBuilder.Build(ctx, orgID, start, end, requestType, gated, variables)
|
||||
}
|
||||
|
||||
// buildTraceListQuery wires the CTE pipeline: one windowed pass picks the top-N
|
||||
// traces, then a bucket-pruned pass enriches only those.
|
||||
// Helpers appear in this file in the order they run. start/end are nanoseconds.
|
||||
//
|
||||
// RESOLVE (keys/columns → SQL via the field mapper)
|
||||
// fetchKeys metadata for every key we reference
|
||||
// resolveMask the "span is in scope" predicate (OR of EXISTS)
|
||||
// resolveColumns per-trace column SQL
|
||||
// resolveListOrders which columns to ORDER BY
|
||||
// splitFilter span-level predicate + trace-level HAVING
|
||||
//
|
||||
// BUILD
|
||||
// matched one windowed, mask-pruned GROUP BY trace_id scan fusing gate + span
|
||||
// │ filter + HAVING + ORDER BY + LIMIT/OFFSET → the top-N trace_ids
|
||||
// ▼
|
||||
// ranked [start,end] bounds of those traces, from the small summary table
|
||||
// ▼
|
||||
// buckets the ts_bucket_start values they touch, to prune the next scan
|
||||
// ▼
|
||||
// enrichment every per-trace column for those traces over their full extent
|
||||
// (not window-clipped), scanning only their buckets
|
||||
//
|
||||
// Only Orderable columns are computable in the mask-pruned matched pass, so only they
|
||||
// can be ordered or filtered on; all-span columns (span_count, …) are output-only.
|
||||
func (b *scopedTraceStatementBuilder) buildTraceListQuery(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
|
||||
variables map[string]qbtypes.VariableItem,
|
||||
) (*qbtypes.Statement, error) {
|
||||
|
||||
startBucket := start/querybuilder.NsToSeconds - querybuilder.BucketAdjustment
|
||||
endBucket := end / querybuilder.NsToSeconds
|
||||
|
||||
limit := query.Limit
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
|
||||
// Resolve keys and columns once; all attribute access goes through the field mapper.
|
||||
keys, err := b.fetchKeys(ctx, orgID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mapper := newFieldMapper(b.fm, b.cb, keys)
|
||||
maskExpr, maskArgs, err := b.resolveMask(ctx, orgID, start, end, mapper)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mapper.maskExpr, mapper.maskArgs = maskExpr, maskArgs
|
||||
resolved, err := b.resolveColumns(ctx, orgID, start, end, mapper)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
orders, err := b.resolveListOrders(query.Order, resolved)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
orderableSet := orderableAliasSet(resolved)
|
||||
|
||||
// If the filter references resource attributes, add a __resource_filter CTE and
|
||||
// narrow the matched scan by resource_fingerprint; the span predicate drops those
|
||||
// keys so they aren't applied twice.
|
||||
resourceFrag, resourceArgs, resourcePred, err := b.maybeAttachResourceFilter(ctx, orgID, query, start, end, variables)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Split the user filter: span-level predicate + trace-level HAVING expression.
|
||||
fp, err := b.splitFilter(ctx, orgID, query, b.aggregateAliasSet(), orderableSet, start, end, variables)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// matched → ranked → buckets → enrichment
|
||||
matchedFrag, matchedArgs, err := b.buildMatchedCTE(start, end, startBucket, endBucket, resolved, orders, orderableSet, maskExpr, maskArgs, fp, resourcePred, limit, query.Offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rankedFrag, rankedArgs := b.buildRankedCTE(start, end)
|
||||
bucketsFrag := buildBucketsCTE()
|
||||
mainSQL, mainArgs := b.buildEnrichmentSelect(resolved, orders)
|
||||
|
||||
cteFragments := []string{matchedFrag, rankedFrag, bucketsFrag}
|
||||
cteArgs := [][]any{matchedArgs, rankedArgs, nil}
|
||||
|
||||
// __resource_filter must precede `matched`, which references it.
|
||||
if resourceFrag != "" {
|
||||
cteFragments = append([]string{resourceFrag}, cteFragments...)
|
||||
cteArgs = append([][]any{resourceArgs}, cteArgs...)
|
||||
}
|
||||
|
||||
finalSQL := querybuilder.CombineCTEs(cteFragments) + mainSQL + " SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000"
|
||||
finalArgs := querybuilder.PrependArgs(cteArgs, mainArgs)
|
||||
|
||||
return &qbtypes.Statement{
|
||||
Query: finalSQL,
|
||||
Args: finalArgs,
|
||||
Warnings: fp.warnings,
|
||||
WarningsDocURL: fp.warningsURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// maybeAttachResourceFilter builds the __resource_filter CTE (fingerprints matching
|
||||
// the filter's resource conditions) and the predicate narrowing the span scan by
|
||||
// resource_fingerprint; with no resource conditions it returns empty fragments.
|
||||
//
|
||||
// Unlike the standard trace builder there is deliberately no skip-fingerprint
|
||||
// fallback: falling back would leave the resource conditions inside the OR'd
|
||||
// span-filter bucket, which changes trace membership (any span from the resource +
|
||||
// any gen_ai span, instead of a gen_ai span from the resource). Resource conditions
|
||||
// always scope the whole matched scan.
|
||||
func (b *scopedTraceStatementBuilder) maybeAttachResourceFilter(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
|
||||
start, end uint64,
|
||||
variables map[string]qbtypes.VariableItem,
|
||||
) (cteFrag string, cteArgs []any, fingerprintPred string, err error) {
|
||||
stmt, err := b.resourceFilterStmtBuilder.Build(
|
||||
ctx, orgID, start, end, qbtypes.RequestTypeRaw, query, variables,
|
||||
)
|
||||
if err != nil {
|
||||
return "", nil, "", err
|
||||
}
|
||||
if stmt == nil {
|
||||
return "", nil, "", nil
|
||||
}
|
||||
return fmt.Sprintf("__resource_filter AS (%s)", stmt.Query), stmt.Args,
|
||||
"resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter)", nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// RESOLVE — turn keys/columns into field-mapper-aware SQL
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func (b *scopedTraceStatementBuilder) fetchKeys(ctx context.Context, orgID valuer.UUID) (map[string][]*telemetrytypes.TelemetryFieldKey, error) {
|
||||
fields := b.resolverFieldKeys()
|
||||
selectors := make([]*telemetrytypes.FieldKeySelector, 0, len(fields))
|
||||
for _, k := range fields {
|
||||
selectors = append(selectors, &telemetrytypes.FieldKeySelector{
|
||||
Name: k.Name,
|
||||
Signal: k.Signal,
|
||||
FieldContext: k.FieldContext,
|
||||
SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact,
|
||||
})
|
||||
}
|
||||
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, selectors)
|
||||
return keys, err
|
||||
}
|
||||
|
||||
func (b *scopedTraceStatementBuilder) resolverFieldKeys() []*telemetrytypes.TelemetryFieldKey {
|
||||
seen := make(map[string]struct{})
|
||||
var out []*telemetrytypes.TelemetryFieldKey
|
||||
add := func(k *telemetrytypes.TelemetryFieldKey) {
|
||||
if k == nil {
|
||||
return
|
||||
}
|
||||
if _, dup := seen[k.Name]; dup {
|
||||
return
|
||||
}
|
||||
seen[k.Name] = struct{}{}
|
||||
out = append(out, k)
|
||||
}
|
||||
for _, k := range b.scope.FieldKeys {
|
||||
add(k)
|
||||
}
|
||||
for _, c := range b.scope.Columns {
|
||||
for _, k := range c.Expr.keys {
|
||||
add(k)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// resolveMask builds the per-span in-scope mask: OR of resolved EXISTS predicates
|
||||
// over the base condition's field keys.
|
||||
func (b *scopedTraceStatementBuilder) resolveMask(ctx context.Context, orgID valuer.UUID, start, end uint64, mapper *fieldMapper) (string, []any, error) {
|
||||
fieldKeys := b.scope.FieldKeys
|
||||
parts := make([]string, 0, len(fieldKeys))
|
||||
var args []any
|
||||
for _, key := range fieldKeys {
|
||||
e, a, err := mapper.ExistsFor(ctx, orgID, start, end, key)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
parts = append(parts, e)
|
||||
args = append(args, a...)
|
||||
}
|
||||
return "(" + strings.Join(parts, " OR ") + ")", args, nil
|
||||
}
|
||||
|
||||
// resolvedColumn is a column resolved to SQL via the field mapper; expr is escaped
|
||||
// once, ready to embed in an outer SELECT.
|
||||
type resolvedColumn struct {
|
||||
alias string
|
||||
expr string
|
||||
args []any
|
||||
orderable bool
|
||||
}
|
||||
|
||||
// resolveColumns turns the declarative columns into SQL through the resolver, so all
|
||||
// attribute access goes through the field mapper / condition builder.
|
||||
func (b *scopedTraceStatementBuilder) resolveColumns(ctx context.Context, orgID valuer.UUID, start, end uint64, mapper *fieldMapper) ([]resolvedColumn, error) {
|
||||
cols := b.scope.Columns
|
||||
out := make([]resolvedColumn, 0, len(cols))
|
||||
for _, c := range cols {
|
||||
expr, args, err := c.Expr.render(ctx, orgID, start, end, mapper)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, resolvedColumn{alias: c.Alias, expr: expr, args: args, orderable: c.Orderable})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// listOrder is a sort key resolved to a column alias + direction; both the matched
|
||||
// CTE and the enrichment ORDER BY it.
|
||||
type listOrder struct {
|
||||
alias string
|
||||
direction string
|
||||
}
|
||||
|
||||
// resolveListOrders maps order keys to the resolved orderable columns; non-orderable
|
||||
// columns are rejected. Defaults to the column provider's default order.
|
||||
func (b *scopedTraceStatementBuilder) resolveListOrders(order []qbtypes.OrderBy, resolved []resolvedColumn) ([]listOrder, error) {
|
||||
byAlias := make(map[string]resolvedColumn, len(resolved))
|
||||
orderable := make([]string, 0, len(resolved))
|
||||
for _, rc := range resolved {
|
||||
byAlias[rc.alias] = rc
|
||||
if rc.orderable {
|
||||
orderable = append(orderable, rc.alias)
|
||||
}
|
||||
}
|
||||
|
||||
if len(order) == 0 {
|
||||
return []listOrder{{alias: b.scope.DefaultOrderAlias, direction: "DESC"}}, nil
|
||||
}
|
||||
|
||||
orders := make([]listOrder, 0, len(order))
|
||||
for _, o := range order {
|
||||
direction := "DESC"
|
||||
if o.Direction == qbtypes.OrderDirectionAsc {
|
||||
direction = "ASC"
|
||||
}
|
||||
rc, ok := byAlias[o.Key.Name]
|
||||
if !ok || !rc.orderable {
|
||||
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput,
|
||||
"unsupported order key %q for the trace list; orderable keys: %s", o.Key.Name, strings.Join(orderable, ", "))
|
||||
}
|
||||
orders = append(orders, listOrder{alias: rc.alias, direction: direction})
|
||||
}
|
||||
return orders, nil
|
||||
}
|
||||
|
||||
// filterParts is the user filter split into a span-level predicate (widens the
|
||||
// matched WHERE prune and becomes a countIf existence check in HAVING) and a
|
||||
// trace-level HAVING expression.
|
||||
type filterParts struct {
|
||||
spanPred string
|
||||
spanArgs []any
|
||||
hasSpanFilter bool
|
||||
havingExpr string
|
||||
warnings []string
|
||||
warningsURL string
|
||||
}
|
||||
|
||||
// splitFilter splits query.Filter into a span-level predicate and a trace-level
|
||||
// HAVING expression (an explicit query.Having is ANDed onto the latter), then
|
||||
// validates the trace-level part against the matched-pass aggregates.
|
||||
func (b *scopedTraceStatementBuilder) splitFilter(ctx context.Context, orgID valuer.UUID, query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation], classifySet, orderableSet map[string]struct{}, start, end uint64, variables map[string]qbtypes.VariableItem) (filterParts, error) {
|
||||
var fp filterParts
|
||||
if query.Filter != nil && strings.TrimSpace(query.Filter.Expression) != "" {
|
||||
spanExpr, traceExpr, err := querybuilder.SplitFilterForAggregates(query.Filter.Expression, classifySet)
|
||||
if err != nil {
|
||||
return fp, err
|
||||
}
|
||||
fp.havingExpr = traceExpr
|
||||
if strings.TrimSpace(spanExpr) != "" {
|
||||
pred, args, warnings, url, err := b.resolveSpanPredicate(ctx, orgID, start, end, spanExpr, variables)
|
||||
if err != nil {
|
||||
return fp, err
|
||||
}
|
||||
// pred is empty when the span-level keys were all resource attributes
|
||||
// already handled by the __resource_filter CTE.
|
||||
if strings.TrimSpace(pred) != "" {
|
||||
fp.spanPred, fp.spanArgs, fp.hasSpanFilter = pred, args, true
|
||||
}
|
||||
fp.warnings, fp.warningsURL = warnings, url
|
||||
}
|
||||
}
|
||||
if query.Having != nil && strings.TrimSpace(query.Having.Expression) != "" {
|
||||
if fp.havingExpr != "" {
|
||||
fp.havingExpr = fmt.Sprintf("(%s) AND (%s)", fp.havingExpr, query.Having.Expression)
|
||||
} else {
|
||||
fp.havingExpr = query.Having.Expression
|
||||
}
|
||||
}
|
||||
// The span predicate binds variables via PrepareWhereClause; the HAVING is a plain
|
||||
// text rewrite, so substitute variables here (list/IN quoting, __all__ drops the
|
||||
// condition) before validating.
|
||||
if strings.TrimSpace(fp.havingExpr) != "" && len(variables) > 0 {
|
||||
replaced, err := qbvariables.ReplaceVariablesInExpression(fp.havingExpr, variables)
|
||||
if err != nil {
|
||||
return fp, err
|
||||
}
|
||||
fp.havingExpr = replaced
|
||||
}
|
||||
if err := validateAggregateFilter(fp.havingExpr, orderableSet); err != nil {
|
||||
return fp, err
|
||||
}
|
||||
return fp, nil
|
||||
}
|
||||
|
||||
// resolveSpanPredicate resolves a span-level filter expression to a bare boolean
|
||||
// SQL predicate + args via the field mapper.
|
||||
func (b *scopedTraceStatementBuilder) resolveSpanPredicate(ctx context.Context, orgID valuer.UUID, start, end uint64, expr string, variables map[string]qbtypes.VariableItem) (string, []any, []string, string, error) {
|
||||
selectors := querybuilder.QueryStringToKeysSelectors(expr)
|
||||
for i := range selectors {
|
||||
selectors[i].Signal = telemetrytypes.SignalTraces
|
||||
}
|
||||
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, selectors)
|
||||
if err != nil {
|
||||
return "", nil, nil, "", err
|
||||
}
|
||||
prepared, err := querybuilder.PrepareWhereClause(expr, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fm,
|
||||
ConditionBuilder: b.cb,
|
||||
FieldKeys: keys,
|
||||
// resource conditions are always handled by the __resource_filter CTE
|
||||
SkipResourceFilter: true,
|
||||
Variables: variables,
|
||||
StartNs: start,
|
||||
EndNs: end,
|
||||
})
|
||||
if err != nil {
|
||||
return "", nil, nil, "", err
|
||||
}
|
||||
if prepared.IsEmpty() {
|
||||
return "", nil, nil, "", nil
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.AddWhereClause(prepared.WhereClause)
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
pred := sql[strings.Index(sql, "WHERE ")+len("WHERE "):]
|
||||
return sqlbuilder.Escape(pred), args, prepared.Warnings, prepared.WarningsDocURL, nil
|
||||
}
|
||||
|
||||
// buildMatchedCTE builds `matched`: the single windowed GROUP BY trace_id scan that
|
||||
// fuses gate + span filter + HAVING + ORDER BY + LIMIT/OFFSET, selecting only the
|
||||
// aliases the ORDER BY / HAVING reference.
|
||||
func (b *scopedTraceStatementBuilder) buildMatchedCTE(start, end, startBucket, endBucket uint64, resolved []resolvedColumn, orders []listOrder, orderableSet map[string]struct{}, maskExpr string, maskArgs []any, fp filterParts, resourcePred string, limit, offset int) (string, []any, error) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
// SELECT trace_id + only the aggregates ORDER BY / HAVING reference (as aliases).
|
||||
needed := neededMatchedAliases(orders, fp.havingExpr, orderableSet)
|
||||
selects := []string{"trace_id"}
|
||||
for _, rc := range resolved {
|
||||
if _, ok := needed[rc.alias]; !ok {
|
||||
continue
|
||||
}
|
||||
colExpr, err := embedExpr(sb, rc.expr, rc.args)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
selects = append(selects, colExpr+" AS "+quoteAlias(rc.alias))
|
||||
}
|
||||
sb.Select(selects...)
|
||||
sb.From(spanTable())
|
||||
|
||||
// WHERE: window + prune to in-scope spans, widened by the span filter so its
|
||||
// spans survive for the countIf existence check below.
|
||||
win := windowWhere(sb, start, end, startBucket, endBucket)
|
||||
mask, err := embedExpr(sb, maskExpr, maskArgs)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
prune := "(" + mask
|
||||
if fp.hasSpanFilter {
|
||||
spanPred, err := embedExpr(sb, fp.spanPred, fp.spanArgs)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
prune += " OR " + spanPred
|
||||
}
|
||||
prune += ")"
|
||||
where := append(win, prune)
|
||||
if resourcePred != "" {
|
||||
where = append(where, resourcePred)
|
||||
}
|
||||
sb.Where(where...)
|
||||
sb.GroupBy("trace_id")
|
||||
|
||||
// HAVING: the gate/span existence checks are only needed when the WHERE was
|
||||
// widened by a span filter; otherwise the mask alone already enforces the gate.
|
||||
var having []string
|
||||
if fp.hasSpanFilter {
|
||||
havingMask, err := embedExpr(sb, maskExpr, maskArgs)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
havingPred, err := embedExpr(sb, fp.spanPred, fp.spanArgs)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
having = append(having, "countIf("+havingMask+") > 0")
|
||||
having = append(having, "countIf("+havingPred+") > 0")
|
||||
}
|
||||
if strings.TrimSpace(fp.havingExpr) != "" {
|
||||
hv, err := b.buildHaving(fp.havingExpr, orderableSet)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if hv != "" {
|
||||
// hv carries user text with values inlined by the rewriter; escape it so a
|
||||
// literal $ can't be read as an interpolation marker at Build time. The
|
||||
// countIf entries above hold live placeholders and must stay unescaped.
|
||||
having = append(having, sqlbuilder.Escape(hv))
|
||||
}
|
||||
}
|
||||
if len(having) > 0 {
|
||||
sb.Having(strings.Join(having, " AND "))
|
||||
}
|
||||
|
||||
sb.OrderBy(orderClause(orders)...)
|
||||
sb.Limit(limit)
|
||||
if offset > 0 {
|
||||
sb.Offset(offset)
|
||||
}
|
||||
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
return fmt.Sprintf("matched AS (%s)", sql), args, nil
|
||||
}
|
||||
|
||||
// buildRankedCTE builds `ranked`: [start,end] bounds per matched trace, read from the
|
||||
// small trace-summary table.
|
||||
func (b *scopedTraceStatementBuilder) buildRankedCTE(start, end uint64) (string, []any) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("trace_id", "min(start) AS t_start", "max(end) AS t_end")
|
||||
sb.From(summaryTable())
|
||||
sb.Where(
|
||||
"trace_id GLOBAL IN (SELECT trace_id FROM matched)",
|
||||
"end >= fromUnixTimestamp64Nano("+sb.Var(start)+")",
|
||||
"start < fromUnixTimestamp64Nano("+sb.Var(end)+")",
|
||||
)
|
||||
sb.GroupBy("trace_id")
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
return fmt.Sprintf("ranked AS (%s)", sql), args
|
||||
}
|
||||
|
||||
// buildBucketsCTE builds `buckets`: the ts_bucket_start values the matched traces
|
||||
// span, so the enrichment scan is primary-key pruned. No args.
|
||||
func buildBucketsCTE() string {
|
||||
adj := querybuilder.BucketAdjustment // 30-min bucket width in seconds
|
||||
return fmt.Sprintf("buckets AS (SELECT DISTINCT b AS ts_bucket FROM ranked "+
|
||||
"ARRAY JOIN range("+
|
||||
"toUInt64(intDiv(toUnixTimestamp(t_start), %d) * %d - %d), "+
|
||||
"toUInt64(intDiv(toUnixTimestamp(t_end), %d) * %d + %d), "+
|
||||
"%d) AS b)", adj, adj, adj, adj, adj, adj, adj)
|
||||
}
|
||||
|
||||
// buildEnrichmentSelect builds the final SELECT: every per-trace column for the
|
||||
// matched traces over their full extent, scanning only their buckets.
|
||||
//
|
||||
// Accepted discrepancy: matched ranks/paginates on window-clipped values (and, with a
|
||||
// resource filter, only over fingerprint-matching spans), while this pass recomputes
|
||||
// and ORDER BYs full-trace values — so a trace with activity outside the window or
|
||||
// resource can sort differently than it ranked. Page membership is unaffected
|
||||
// (LIMIT/OFFSET runs only in matched); rows still sort by the values the user sees.
|
||||
// Ordering by matched's values instead would re-run the matched scan (ClickHouse
|
||||
// re-executes a CTE per reference) without fixing the visible cross-page artifact.
|
||||
func (b *scopedTraceStatementBuilder) buildEnrichmentSelect(resolved []resolvedColumn, orders []listOrder) (string, []any) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
selects, selectArgs := selectAllColumns(resolved)
|
||||
sb.Select(selects...)
|
||||
sb.From(spanTable())
|
||||
sb.Where(
|
||||
"ts_bucket_start GLOBAL IN (SELECT ts_bucket FROM buckets)",
|
||||
"trace_id GLOBAL IN (SELECT trace_id FROM ranked)",
|
||||
)
|
||||
sb.GroupBy("trace_id")
|
||||
sb.OrderBy(orderClause(orders)...)
|
||||
sql, builtArgs := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
return sql, append(append([]any{}, selectArgs...), builtArgs...)
|
||||
}
|
||||
|
||||
// buildHaving rewrites a trace-level HAVING expression to the matched-pass column
|
||||
// aliases. The rewriter matches raw key text, so the trace. form is mapped alongside
|
||||
// the bare name.
|
||||
func (b *scopedTraceStatementBuilder) buildHaving(havingExpr string, orderableSet map[string]struct{}) (string, error) {
|
||||
columnMap := make(map[string]string, len(orderableSet)*2)
|
||||
for a := range orderableSet {
|
||||
columnMap[a] = quoteAlias(a)
|
||||
columnMap[telemetrytypes.FieldContextTrace.StringValue()+"."+a] = quoteAlias(a)
|
||||
}
|
||||
return querybuilder.NewHavingExpressionRewriter().Rewrite(havingExpr, columnMap)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Small shared SQL-builder utilities
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// spanTable is the fully-qualified span index table.
|
||||
func spanTable() string {
|
||||
return fmt.Sprintf("%s.%s", tracestelemetryschema.DBName, tracestelemetryschema.SpanIndexV3TableName)
|
||||
}
|
||||
|
||||
// summaryTable is the fully-qualified trace-summary table.
|
||||
func summaryTable() string {
|
||||
return fmt.Sprintf("%s.%s", tracestelemetryschema.DBName, tracestelemetryschema.TraceSummaryTableName)
|
||||
}
|
||||
|
||||
// aggregateAliasSet is every trace-level column alias, used to classify filter keys
|
||||
// as trace-level vs span-level. Derived from the scope's columns so a new column
|
||||
// can't be forgotten; SpanLevel columns are filtered span-level, so skip them.
|
||||
func (b *scopedTraceStatementBuilder) aggregateAliasSet() map[string]struct{} {
|
||||
set := make(map[string]struct{}, len(b.scope.Columns))
|
||||
for _, c := range b.scope.Columns {
|
||||
if !c.SpanLevel {
|
||||
set[c.Alias] = struct{}{}
|
||||
}
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
// orderableAliasSet is the subset of aliases computable in the matched pass — the
|
||||
// only ones usable in ORDER BY and the aggregate filter.
|
||||
func orderableAliasSet(resolved []resolvedColumn) map[string]struct{} {
|
||||
set := make(map[string]struct{})
|
||||
for _, rc := range resolved {
|
||||
if rc.orderable {
|
||||
set[rc.alias] = struct{}{}
|
||||
}
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
// neededMatchedAliases is the minimal alias set the matched pass must select: those
|
||||
// in ORDER BY plus those in the aggregate HAVING. Everything else is left to the
|
||||
// enrichment scan.
|
||||
func neededMatchedAliases(orders []listOrder, havingExpr string, orderableSet map[string]struct{}) map[string]struct{} {
|
||||
needed := make(map[string]struct{})
|
||||
for _, o := range orders {
|
||||
needed[o.alias] = struct{}{}
|
||||
}
|
||||
for _, name := range traceAggregateNames(havingExpr) {
|
||||
if _, ok := orderableSet[name]; ok {
|
||||
needed[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
return needed
|
||||
}
|
||||
|
||||
// traceAggregateNames extracts the aggregate names a trace-level HAVING expression
|
||||
// references. QueryStringToKeysSelectors emits an extra attribute-context fallback
|
||||
// selector for context-prefixed keys (`trace.x` → attribute "trace.x"); only the
|
||||
// unspecified- and trace-context selectors name aggregates.
|
||||
func traceAggregateNames(havingExpr string) []string {
|
||||
var names []string
|
||||
for _, sel := range querybuilder.QueryStringToKeysSelectors(havingExpr) {
|
||||
if sel.FieldContext == telemetrytypes.FieldContextUnspecified || sel.FieldContext == telemetrytypes.FieldContextTrace {
|
||||
names = append(names, sel.Name)
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// validateAggregateFilter rejects a trace-level filter referencing an aggregate not
|
||||
// computable in the matched pass (e.g. span_count, trace_duration_nano).
|
||||
func validateAggregateFilter(havingExpr string, orderableSet map[string]struct{}) error {
|
||||
if strings.TrimSpace(havingExpr) == "" {
|
||||
return nil
|
||||
}
|
||||
allowed := make([]string, 0, len(orderableSet))
|
||||
for a := range orderableSet {
|
||||
allowed = append(allowed, a)
|
||||
}
|
||||
sort.Strings(allowed)
|
||||
for _, name := range traceAggregateNames(havingExpr) {
|
||||
if _, ok := orderableSet[name]; !ok {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput,
|
||||
"aggregate %q cannot be used in the trace-list filter; filterable aggregates: %s", name, strings.Join(allowed, ", "))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// embedExpr inlines a resolved expr into sb, replacing each `?` placeholder with a
|
||||
// builder Var so the args are tracked in appearance order. Resolved exprs carry
|
||||
// values only as bound args, so every `?` is a placeholder; a count mismatch would
|
||||
// silently shift args into the wrong slots — error out instead.
|
||||
func embedExpr(sb *sqlbuilder.SelectBuilder, expr string, args []any) (string, error) {
|
||||
if n := strings.Count(expr, "?"); n != len(args) {
|
||||
return "", errors.NewInternalf(errors.CodeInternal,
|
||||
"scoped trace builder: %d placeholders != %d args embedding %q", n, len(args), expr)
|
||||
}
|
||||
var out strings.Builder
|
||||
ai := 0
|
||||
for i := 0; i < len(expr); i++ {
|
||||
if expr[i] == '?' {
|
||||
out.WriteString(sb.Var(args[ai]))
|
||||
ai++
|
||||
continue
|
||||
}
|
||||
out.WriteByte(expr[i])
|
||||
}
|
||||
return out.String(), nil
|
||||
}
|
||||
|
||||
// windowWhere binds the time-window predicates to sb and returns them so the caller
|
||||
// can add its own predicates in the same Where call.
|
||||
func windowWhere(sb *sqlbuilder.SelectBuilder, start, end, startBucket, endBucket uint64) []string {
|
||||
return []string{
|
||||
sb.GE("timestamp", fmt.Sprintf("%d", start)),
|
||||
sb.L("timestamp", fmt.Sprintf("%d", end)),
|
||||
sb.GE("ts_bucket_start", startBucket),
|
||||
sb.LE("ts_bucket_start", endBucket),
|
||||
}
|
||||
}
|
||||
|
||||
// orderClause renders the ORDER BY terms plus the trace_id tiebreak.
|
||||
func orderClause(orders []listOrder) []string {
|
||||
out := make([]string, 0, len(orders)+1)
|
||||
for _, o := range orders {
|
||||
out = append(out, fmt.Sprintf("%s %s", quoteAlias(o.alias), o.direction))
|
||||
}
|
||||
return append(out, "trace_id DESC")
|
||||
}
|
||||
|
||||
// selectAllColumns renders `expr AS alias` for every resolved column, args in select
|
||||
// order.
|
||||
func selectAllColumns(resolved []resolvedColumn) ([]string, []any) {
|
||||
selects := []string{"trace_id"}
|
||||
var args []any
|
||||
for _, rc := range resolved {
|
||||
selects = append(selects, rc.expr+" AS "+quoteAlias(rc.alias))
|
||||
args = append(args, rc.args...)
|
||||
}
|
||||
return selects, args
|
||||
}
|
||||
|
||||
// quoteAlias backticks an alias containing characters special to the SQL builder.
|
||||
func quoteAlias(alias string) string {
|
||||
if strings.ContainsAny(alias, ".$`") {
|
||||
return "`" + alias + "`"
|
||||
}
|
||||
return alias
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package scopedtracesstatementbuilder
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The full-pipeline golden tests live in pkg/statementbuilder/aistatementbuilder,
|
||||
// which exercises this
|
||||
// builder through its production provider pair. The tests here cover only what
|
||||
// needs the package internals.
|
||||
|
||||
// embedExpr treats every `?` byte as a placeholder; a count/args mismatch (an expr
|
||||
// carrying a literal `?`, or a dropped arg) must fail loudly instead of silently
|
||||
// shifting every subsequent arg into the wrong placeholder.
|
||||
func TestEmbedExpr_PlaceholderArgMismatch(t *testing.T) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
out, err := embedExpr(sb, "x = ? AND y = ?", []any{1, 2})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, strings.Count(out, "$"), "both placeholders bound as builder vars")
|
||||
|
||||
_, err = embedExpr(sb, "x = ? AND y LIKE 'a?b'", []any{1})
|
||||
require.Error(t, err, "literal ? in the expr must not pass as a placeholder")
|
||||
|
||||
_, err = embedExpr(sb, "x = ?", []any{1, 2})
|
||||
require.Error(t, err, "extra args must not be silently dropped")
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrytraces
|
||||
package tracesstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,7 +10,9 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryresourcefilter"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/resourcefilter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -27,13 +29,35 @@ type traceQueryStatementBuilder struct {
|
||||
metadataStore telemetrytypes.MetadataStore
|
||||
fm qbtypes.FieldMapper
|
||||
cb qbtypes.ConditionBuilder
|
||||
resourceFilterResolver *telemetryresourcefilter.ResourceFingerprintResolver[qbtypes.TraceAggregation]
|
||||
resourceFilterResolver *resourcefilter.ResourceFingerprintResolver[qbtypes.TraceAggregation]
|
||||
aggExprRewriter qbtypes.AggExprRewriter
|
||||
skipResourceFingerprintEnabled bool
|
||||
}
|
||||
|
||||
var _ qbtypes.StatementBuilder[qbtypes.TraceAggregation] = (*traceQueryStatementBuilder)(nil)
|
||||
|
||||
// NewFactory returns a provider factory for the trace query statement builder. Its
|
||||
// New internalizes the FieldMapper, ConditionBuilder, and AggExprRewriter, and reads
|
||||
// SkipResourceFingerprint from the config.
|
||||
func NewFactory(
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
fl flagger.Flagger,
|
||||
) factory.ProviderFactory[qbtypes.StatementBuilder[qbtypes.TraceAggregation], statementbuilder.Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("traces"),
|
||||
func(_ context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.StatementBuilder[qbtypes.TraceAggregation], error) {
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, fm, cb, fl)
|
||||
return NewTraceQueryStatementBuilder(
|
||||
settings, metadataStore, fm, cb, aggExprRewriter, telemetryStore, fl,
|
||||
cfg.SkipResourceFingerprint.Enabled, cfg.SkipResourceFingerprint.Threshold,
|
||||
), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func NewTraceQueryStatementBuilder(
|
||||
settings factory.ProviderSettings,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
@@ -45,12 +69,12 @@ func NewTraceQueryStatementBuilder(
|
||||
skipResourceFingerprintEnable bool,
|
||||
skipResourceFingerprintThreshold uint64,
|
||||
) *traceQueryStatementBuilder {
|
||||
tracesSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrytraces")
|
||||
tracesSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema")
|
||||
|
||||
resourceFilterResolver := telemetryresourcefilter.NewResolver[qbtypes.TraceAggregation](
|
||||
resourceFilterResolver := resourcefilter.NewResolver[qbtypes.TraceAggregation](
|
||||
settings,
|
||||
DBName,
|
||||
TracesResourceV3TableName,
|
||||
tracestelemetryschema.DBName,
|
||||
tracestelemetryschema.TracesResourceV3TableName,
|
||||
telemetrytypes.SignalTraces,
|
||||
telemetrytypes.SourceUnspecified,
|
||||
metadataStore,
|
||||
@@ -183,10 +207,10 @@ func getKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation])
|
||||
// 3. We prepend (not append) so the intrinsic/calculated entry wins ordering
|
||||
// in the multi_if SQL expression.
|
||||
func mergeDeprecatedTraceKeys(keys map[string][]*telemetrytypes.TelemetryFieldKey) {
|
||||
for fieldKeyName, fieldKey := range IntrinsicFieldsDeprecated {
|
||||
for fieldKeyName, fieldKey := range tracestelemetryschema.IntrinsicFieldsDeprecated {
|
||||
keys[fieldKeyName] = append([]*telemetrytypes.TelemetryFieldKey{&fieldKey}, keys[fieldKeyName]...)
|
||||
}
|
||||
for fieldKeyName, fieldKey := range CalculatedFieldsDeprecated {
|
||||
for fieldKeyName, fieldKey := range tracestelemetryschema.CalculatedFieldsDeprecated {
|
||||
keys[fieldKeyName] = append([]*telemetrytypes.TelemetryFieldKey{&fieldKey}, keys[fieldKeyName]...)
|
||||
}
|
||||
}
|
||||
@@ -246,18 +270,18 @@ func adjustTraceKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*te
|
||||
*/
|
||||
var isIntrinsicOrCalculatedField bool
|
||||
var intrinsicOrCalculatedField telemetrytypes.TelemetryFieldKey
|
||||
if _, ok := IntrinsicFields[key.Name]; ok {
|
||||
if _, ok := tracestelemetryschema.IntrinsicFields[key.Name]; ok {
|
||||
isIntrinsicOrCalculatedField = true
|
||||
intrinsicOrCalculatedField = IntrinsicFields[key.Name]
|
||||
} else if _, ok := CalculatedFields[key.Name]; ok {
|
||||
intrinsicOrCalculatedField = tracestelemetryschema.IntrinsicFields[key.Name]
|
||||
} else if _, ok := tracestelemetryschema.CalculatedFields[key.Name]; ok {
|
||||
isIntrinsicOrCalculatedField = true
|
||||
intrinsicOrCalculatedField = CalculatedFields[key.Name]
|
||||
} else if _, ok := IntrinsicFieldsDeprecated[key.Name]; ok {
|
||||
intrinsicOrCalculatedField = tracestelemetryschema.CalculatedFields[key.Name]
|
||||
} else if _, ok := tracestelemetryschema.IntrinsicFieldsDeprecated[key.Name]; ok {
|
||||
isIntrinsicOrCalculatedField = true
|
||||
intrinsicOrCalculatedField = IntrinsicFieldsDeprecated[key.Name]
|
||||
} else if _, ok := CalculatedFieldsDeprecated[key.Name]; ok {
|
||||
intrinsicOrCalculatedField = tracestelemetryschema.IntrinsicFieldsDeprecated[key.Name]
|
||||
} else if _, ok := tracestelemetryschema.CalculatedFieldsDeprecated[key.Name]; ok {
|
||||
isIntrinsicOrCalculatedField = true
|
||||
intrinsicOrCalculatedField = CalculatedFieldsDeprecated[key.Name]
|
||||
intrinsicOrCalculatedField = tracestelemetryschema.CalculatedFieldsDeprecated[key.Name]
|
||||
}
|
||||
|
||||
if isIntrinsicOrCalculatedField {
|
||||
@@ -304,13 +328,13 @@ func (b *traceQueryStatementBuilder) buildListQuery(
|
||||
}
|
||||
|
||||
if isSelectFieldsEmpty {
|
||||
for _, col := range ContextualSpanColumns {
|
||||
for _, col := range tracestelemetryschema.ContextualSpanColumns {
|
||||
sb.SelectMore(col)
|
||||
}
|
||||
}
|
||||
|
||||
// From table
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, SpanIndexV3TableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", tracestelemetryschema.DBName, tracestelemetryschema.SpanIndexV3TableName))
|
||||
|
||||
// Add filter conditions
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables, skipResourceFilter)
|
||||
@@ -368,7 +392,7 @@ func (b *traceQueryStatementBuilder) buildTraceQuery(
|
||||
|
||||
distSB := sqlbuilder.NewSelectBuilder()
|
||||
distSB.Select("trace_id")
|
||||
distSB.From(fmt.Sprintf("%s.%s", DBName, SpanIndexV3TableName))
|
||||
distSB.From(fmt.Sprintf("%s.%s", tracestelemetryschema.DBName, tracestelemetryschema.SpanIndexV3TableName))
|
||||
|
||||
var (
|
||||
cteFragments []string
|
||||
@@ -398,7 +422,7 @@ func (b *traceQueryStatementBuilder) buildTraceQuery(
|
||||
// Build the inner subquery for root spans
|
||||
innerSB := sqlbuilder.NewSelectBuilder()
|
||||
innerSB.Select("trace_id", "duration_nano", sqlbuilder.Escape("resource_string_service$$name as `service.name`"), "name")
|
||||
innerSB.From(fmt.Sprintf("%s.%s", DBName, SpanIndexV3TableName))
|
||||
innerSB.From(fmt.Sprintf("%s.%s", tracestelemetryschema.DBName, tracestelemetryschema.SpanIndexV3TableName))
|
||||
innerSB.Where("parent_span_id = ''")
|
||||
|
||||
// this only helps when there is a filter
|
||||
@@ -528,7 +552,7 @@ func (b *traceQueryStatementBuilder) buildTimeSeriesQuery(
|
||||
sb.SelectMore(fmt.Sprintf("%s AS __result_%d", rewritten, i))
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, SpanIndexV3TableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", tracestelemetryschema.DBName, tracestelemetryschema.SpanIndexV3TableName))
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables, skipResourceFilter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -690,7 +714,7 @@ func (b *traceQueryStatementBuilder) buildScalarQuery(
|
||||
}
|
||||
|
||||
// From table
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, SpanIndexV3TableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", tracestelemetryschema.DBName, tracestelemetryschema.SpanIndexV3TableName))
|
||||
|
||||
// Add filter conditions
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables, skipResourceFilter)
|
||||
@@ -873,21 +897,21 @@ func (b *traceQueryStatementBuilder) maybeAttachResourceFilter(
|
||||
// It must be called before adjustKeys so that normalization runs over the full set.
|
||||
func (b *traceQueryStatementBuilder) expandRawSelectFields(query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]) qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation] {
|
||||
if len(query.SelectFields) == 0 {
|
||||
selectFields := make([]telemetrytypes.TelemetryFieldKey, 0, len(IntrinsicSpanFields)+len(CalculatedSpanFields))
|
||||
selectFields = append(selectFields, IntrinsicSpanFields...)
|
||||
selectFields = append(selectFields, CalculatedSpanFields...)
|
||||
selectFields := make([]telemetrytypes.TelemetryFieldKey, 0, len(tracestelemetryschema.IntrinsicSpanFields)+len(tracestelemetryschema.CalculatedSpanFields))
|
||||
selectFields = append(selectFields, tracestelemetryschema.IntrinsicSpanFields...)
|
||||
selectFields = append(selectFields, tracestelemetryschema.CalculatedSpanFields...)
|
||||
query.SelectFields = selectFields
|
||||
return query
|
||||
}
|
||||
|
||||
selectFields := []telemetrytypes.TelemetryFieldKey{
|
||||
{Name: SpanTimestampColumn, FieldContext: telemetrytypes.FieldContextSpan},
|
||||
{Name: SpanTraceIDColumn, FieldContext: telemetrytypes.FieldContextSpan},
|
||||
{Name: SpanSpanIDColumn, FieldContext: telemetrytypes.FieldContextSpan},
|
||||
{Name: tracestelemetryschema.SpanTimestampColumn, FieldContext: telemetrytypes.FieldContextSpan},
|
||||
{Name: tracestelemetryschema.SpanTraceIDColumn, FieldContext: telemetrytypes.FieldContextSpan},
|
||||
{Name: tracestelemetryschema.SpanSpanIDColumn, FieldContext: telemetrytypes.FieldContextSpan},
|
||||
}
|
||||
for _, field := range query.SelectFields {
|
||||
// TODO(tvats): If a user specifies attribute.timestamp in the select fields, this loop will basically ignore it, as we already added a field by default. This can be fixed once we close https://github.com/SigNoz/engineering-pod/issues/3693
|
||||
if field.Name == SpanTimestampColumn || field.Name == SpanTraceIDColumn || field.Name == SpanSpanIDColumn {
|
||||
if field.Name == tracestelemetryschema.SpanTimestampColumn || field.Name == tracestelemetryschema.SpanTraceIDColumn || field.Name == tracestelemetryschema.SpanSpanIDColumn {
|
||||
continue
|
||||
}
|
||||
selectFields = append(selectFields, field)
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrytraces
|
||||
package tracesstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -375,10 +376,10 @@ func TestStatementBuilder(t *testing.T) {
|
||||
}
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
statementBuilder := NewTraceQueryStatementBuilder(
|
||||
@@ -676,10 +677,10 @@ func TestStatementBuilderListQuery(t *testing.T) {
|
||||
}
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
statementBuilder := NewTraceQueryStatementBuilder(
|
||||
@@ -740,7 +741,7 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Evolutions: mockEvolutionData(time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)),
|
||||
Evolutions: tracestelemetryschema.MockEvolutionData(time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -774,7 +775,7 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Evolutions: mockEvolutionData(time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)),
|
||||
Evolutions: tracestelemetryschema.MockEvolutionData(time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -803,12 +804,12 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
fl := flaggertest.New(t)
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = c.keysMap
|
||||
if mockMetadataStore.KeysMap == nil {
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
}
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
@@ -878,10 +879,10 @@ func TestStatementBuilderGroupByResourceEvolution(t *testing.T) {
|
||||
}
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
statementBuilder := NewTraceQueryStatementBuilder(
|
||||
@@ -1045,10 +1046,10 @@ func TestStatementBuilderTraceQuery(t *testing.T) {
|
||||
}
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
statementBuilder := NewTraceQueryStatementBuilder(
|
||||
@@ -1096,8 +1097,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: IntrinsicFields["trace_id"],
|
||||
keysMap: tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: tracestelemetryschema.IntrinsicFields["trace_id"],
|
||||
},
|
||||
{
|
||||
name: "intrinsic field with metadata match with incorrect context - override",
|
||||
@@ -1106,7 +1107,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextBody, // incorrect context
|
||||
FieldDataType: telemetrytypes.FieldDataTypeInt64,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "duration_nano",
|
||||
FieldContext: telemetrytypes.FieldContextSpan, // should be corrected
|
||||
@@ -1120,7 +1121,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextSpan, // correct context
|
||||
FieldDataType: telemetrytypes.FieldDataTypeInt64,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "duration_nano",
|
||||
FieldContext: telemetrytypes.FieldContextSpan, // should be corrected
|
||||
@@ -1134,8 +1135,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["service.name"][0],
|
||||
keysMap: tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["service.name"][0],
|
||||
},
|
||||
{
|
||||
name: "single matching key with context specified - override",
|
||||
@@ -1144,8 +1145,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["cart.items_count"][0],
|
||||
keysMap: tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["cart.items_count"][0],
|
||||
},
|
||||
{
|
||||
name: "multiple matching keys - all materialized",
|
||||
@@ -1182,7 +1183,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "mixed.materialization.key",
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
@@ -1196,7 +1197,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "mixed.materialization.key",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
@@ -1211,7 +1212,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "unknown.field",
|
||||
Materialized: false,
|
||||
@@ -1224,7 +1225,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
@@ -1239,7 +1240,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "cart.items_count",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
@@ -1254,7 +1255,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "user.id",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
@@ -1308,7 +1309,7 @@ func TestAdjustKeys(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedSelectFields: []telemetrytypes.TelemetryFieldKey{
|
||||
{
|
||||
Name: "service.name",
|
||||
@@ -1345,7 +1346,7 @@ func TestAdjustKeys(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedGroupBy: []qbtypes.GroupByKey{
|
||||
{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
@@ -1392,7 +1393,7 @@ func TestAdjustKeys(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedOrder: []qbtypes.OrderBy{
|
||||
{
|
||||
Key: qbtypes.OrderByKey{
|
||||
@@ -1451,7 +1452,7 @@ func TestAdjustKeys(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedSelectFields: []telemetrytypes.TelemetryFieldKey{
|
||||
{
|
||||
Name: "trace_id",
|
||||
@@ -1506,7 +1507,7 @@ func TestAdjustKeys(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
// After alias adjustment, name becomes "span.duration" with FieldContextUnspecified
|
||||
// "span.duration" is not in keysMap, so context stays unspecified
|
||||
expectedOrder: []qbtypes.OrderBy{
|
||||
@@ -1574,14 +1575,14 @@ func TestAdjustKeys(t *testing.T) {
|
||||
if c.expectDeprecatedFieldsAdd {
|
||||
// Check that at least some deprecated fields were added
|
||||
foundDeprecatedField := false
|
||||
for fieldName := range IntrinsicFieldsDeprecated {
|
||||
for fieldName := range tracestelemetryschema.IntrinsicFieldsDeprecated {
|
||||
if _, ok := keysMapCopy[fieldName]; ok {
|
||||
foundDeprecatedField = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !foundDeprecatedField {
|
||||
for fieldName := range CalculatedFieldsDeprecated {
|
||||
for fieldName := range tracestelemetryschema.CalculatedFieldsDeprecated {
|
||||
if _, ok := keysMapCopy[fieldName]; ok {
|
||||
foundDeprecatedField = true
|
||||
break
|
||||
@@ -1683,11 +1684,11 @@ func newSkipResourceFingerprintBuilder(
|
||||
t.Helper()
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(
|
||||
instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl,
|
||||
@@ -1713,10 +1714,10 @@ func TestStatementBuilderGroupByUnseenKey(t *testing.T) {
|
||||
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
statementBuilder := NewTraceQueryStatementBuilder(
|
||||
@@ -1755,10 +1756,10 @@ func TestStatementBuilderAggregationUnseenKey(t *testing.T) {
|
||||
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
statementBuilder := NewTraceQueryStatementBuilder(
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrytraces
|
||||
package tracesstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -136,7 +137,7 @@ func (b *traceOperatorCTEBuilder) buildAllSpansCTE(ctx context.Context) {
|
||||
sb.Select("*")
|
||||
sb.SelectMore(sqlbuilder.Escape("resource_string_service$$name") + " AS `service.name`")
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, SpanIndexV3TableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", tracestelemetryschema.DBName, tracestelemetryschema.SpanIndexV3TableName))
|
||||
startBucket := b.start/querybuilder.NsToSeconds - querybuilder.BucketAdjustment
|
||||
endBucket := b.end / querybuilder.NsToSeconds
|
||||
sb.Where(
|
||||
@@ -245,7 +246,7 @@ func (b *traceOperatorCTEBuilder) buildQueryCTE(ctx context.Context, queryName s
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("*")
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, SpanIndexV3TableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", tracestelemetryschema.DBName, tracestelemetryschema.SpanIndexV3TableName))
|
||||
if resourceFilterCTEName != "" {
|
||||
sb.Where(fmt.Sprintf("resource_fingerprint GLOBAL IN (SELECT fingerprint FROM %s)", resourceFilterCTEName))
|
||||
}
|
||||
@@ -436,9 +437,9 @@ func (b *traceOperatorCTEBuilder) buildFinalQuery(ctx context.Context, selectFro
|
||||
if requestType == qbtypes.RequestTypeRaw {
|
||||
isSelectFieldsEmpty = len(b.operator.SelectFields) == 0
|
||||
if isSelectFieldsEmpty {
|
||||
b.operator.SelectFields = make([]telemetrytypes.TelemetryFieldKey, 0, len(IntrinsicSpanFields)+len(CalculatedSpanFields))
|
||||
b.operator.SelectFields = append(b.operator.SelectFields, IntrinsicSpanFields...)
|
||||
b.operator.SelectFields = append(b.operator.SelectFields, CalculatedSpanFields...)
|
||||
b.operator.SelectFields = make([]telemetrytypes.TelemetryFieldKey, 0, len(tracestelemetryschema.IntrinsicSpanFields)+len(tracestelemetryschema.CalculatedSpanFields))
|
||||
b.operator.SelectFields = append(b.operator.SelectFields, tracestelemetryschema.IntrinsicSpanFields...)
|
||||
b.operator.SelectFields = append(b.operator.SelectFields, tracestelemetryschema.CalculatedSpanFields...)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -502,7 +503,7 @@ func (b *traceOperatorCTEBuilder) buildListQuery(ctx context.Context, selectFrom
|
||||
}
|
||||
|
||||
if isSelectFieldsEmpty {
|
||||
for _, col := range ContextualSpanColumns {
|
||||
for _, col := range tracestelemetryschema.ContextualSpanColumns {
|
||||
sb.SelectMore(col)
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrytraces
|
||||
package tracesstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
||||
@@ -19,10 +20,10 @@ func newTestTraceOperatorStatementBuilder(t *testing.T) *traceOperatorStatementB
|
||||
t.Helper()
|
||||
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
|
||||
fl := flaggertest.New(t)
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
traceStmtBuilder := NewTraceQueryStatementBuilder(
|
||||
instrumentationtest.New().ToProviderSettings(),
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrytraces
|
||||
package tracesstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -8,7 +8,10 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryresourcefilter"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/resourcefilter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -26,6 +29,32 @@ type traceOperatorStatementBuilder struct {
|
||||
|
||||
var _ qbtypes.TraceOperatorStatementBuilder = (*traceOperatorStatementBuilder)(nil)
|
||||
|
||||
// NewOperatorFactory returns a provider factory for the trace-operator statement
|
||||
// builder. The operator delegates sub-query construction to a trace query statement
|
||||
// builder, so it builds its own internally — mirroring how the meter factory builds
|
||||
// its own metrics builder.
|
||||
func NewOperatorFactory(
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
fl flagger.Flagger,
|
||||
) factory.ProviderFactory[qbtypes.TraceOperatorStatementBuilder, statementbuilder.Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("traceoperator"),
|
||||
func(_ context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.TraceOperatorStatementBuilder, error) {
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, fm, cb, fl)
|
||||
traceStmtBuilder := NewTraceQueryStatementBuilder(
|
||||
settings, metadataStore, fm, cb, aggExprRewriter, telemetryStore, fl,
|
||||
cfg.SkipResourceFingerprint.Enabled, cfg.SkipResourceFingerprint.Threshold,
|
||||
)
|
||||
return NewTraceOperatorStatementBuilder(
|
||||
settings, metadataStore, fm, cb, traceStmtBuilder, aggExprRewriter, fl,
|
||||
), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func NewTraceOperatorStatementBuilder(
|
||||
settings factory.ProviderSettings,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
@@ -35,12 +64,12 @@ func NewTraceOperatorStatementBuilder(
|
||||
aggExprRewriter qbtypes.AggExprRewriter,
|
||||
flagger flagger.Flagger,
|
||||
) *traceOperatorStatementBuilder {
|
||||
tracesSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrytraces")
|
||||
tracesSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema")
|
||||
|
||||
resourceFilterStmtBuilder := telemetryresourcefilter.New[qbtypes.TraceAggregation](
|
||||
resourceFilterStmtBuilder := resourcefilter.New[qbtypes.TraceAggregation](
|
||||
settings,
|
||||
DBName,
|
||||
TracesResourceV3TableName,
|
||||
tracestelemetryschema.DBName,
|
||||
tracestelemetryschema.TracesResourceV3TableName,
|
||||
telemetrytypes.SignalTraces,
|
||||
telemetrytypes.SourceUnspecified,
|
||||
metadataStore,
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrytraces
|
||||
package tracesstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
||||
@@ -20,11 +21,11 @@ import (
|
||||
func TestTraceTimeRangeOptimization(t *testing.T) {
|
||||
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
|
||||
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
mockMetadataStore.KeysMap["trace_id"] = []*telemetrytypes.TelemetryFieldKey{{
|
||||
Name: "trace_id",
|
||||
FieldContext: telemetrytypes.FieldContextSpan,
|
||||
@@ -129,7 +130,7 @@ func TestTraceTimeRangeOptimization(t *testing.T) {
|
||||
assert.NotEmpty(t, stmt.Query)
|
||||
|
||||
if tt.query.Filter != nil && tt.query.Filter.Expression != "" {
|
||||
traceIDs, found := ExtractTraceIDsFromFilter(tt.query.Filter.Expression)
|
||||
traceIDs, found := tracestelemetryschema.ExtractTraceIDsFromFilter(tt.query.Filter.Expression)
|
||||
assert.Equal(t, tt.expectTimeOptimization, found && len(traceIDs) > 0)
|
||||
}
|
||||
})
|
||||
@@ -1,12 +0,0 @@
|
||||
package telemetryaudit
|
||||
|
||||
const (
|
||||
DBName = "signoz_audit"
|
||||
AuditLogsTableName = "distributed_logs"
|
||||
AuditLogsLocalTableName = "logs"
|
||||
TagAttributesTableName = "distributed_tag_attributes"
|
||||
TagAttributesLocalTableName = "tag_attributes"
|
||||
LogAttributeKeysTblName = "distributed_logs_attribute_keys"
|
||||
LogResourceKeysTblName = "distributed_logs_resource_keys"
|
||||
LogsResourceTableName = "distributed_logs_resource"
|
||||
)
|
||||
@@ -1,15 +0,0 @@
|
||||
package telemetrylogs
|
||||
|
||||
const (
|
||||
DBName = "signoz_logs"
|
||||
LogsV2TableName = "distributed_logs_v2"
|
||||
LogsV2LocalTableName = "logs_v2"
|
||||
TagAttributesV2TableName = "distributed_tag_attributes_v2"
|
||||
TagAttributesV2LocalTableName = "tag_attributes_v2"
|
||||
LogAttributeKeysTblName = "distributed_logs_attribute_keys"
|
||||
LogResourceKeysTblName = "distributed_logs_resource_keys"
|
||||
LogsResourceV2TableName = "distributed_logs_v2_resource"
|
||||
PathTypesTableName = "distributed_json_path_types"
|
||||
PromotedPathsTableName = "distributed_json_promoted_paths"
|
||||
SkipIndexTableName = "system.data_skipping_indices"
|
||||
)
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"github.com/SigNoz/signoz-otel-collector/constants"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -139,8 +139,8 @@ func buildListLogsJSONIndexesQuery(cluster string, filters ...string) (string, [
|
||||
"name", "type_full", "expr", "granularity",
|
||||
).From(fmt.Sprintf("clusterAllReplicas('%s', %s)", cluster, SkipIndexTableName))
|
||||
|
||||
sb.Where(sb.Equal("database", telemetrylogs.DBName))
|
||||
sb.Where(sb.Equal("table", telemetrylogs.LogsV2LocalTableName))
|
||||
sb.Where(sb.Equal("database", logstelemetryschema.DBName))
|
||||
sb.Where(sb.Equal("table", logstelemetryschema.LogsV2LocalTableName))
|
||||
sb.Where(sb.Or(
|
||||
sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyV2ColumnPrefix))),
|
||||
sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyPromotedColumnPrefix))),
|
||||
@@ -189,12 +189,12 @@ func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ..
|
||||
baseColumn := ""
|
||||
fieldName := ""
|
||||
switch {
|
||||
case strings.HasPrefix(columnExpr, telemetrylogs.BodyV2ColumnPrefix):
|
||||
baseColumn = telemetrylogs.BodyV2ColumnPrefix
|
||||
fieldName = strings.TrimPrefix(columnExpr, telemetrylogs.BodyV2ColumnPrefix)
|
||||
case strings.HasPrefix(columnExpr, telemetrylogs.BodyPromotedColumnPrefix):
|
||||
baseColumn = telemetrylogs.BodyPromotedColumnPrefix
|
||||
fieldName = strings.TrimPrefix(columnExpr, telemetrylogs.BodyPromotedColumnPrefix)
|
||||
case strings.HasPrefix(columnExpr, logstelemetryschema.BodyV2ColumnPrefix):
|
||||
baseColumn = logstelemetryschema.BodyV2ColumnPrefix
|
||||
fieldName = strings.TrimPrefix(columnExpr, logstelemetryschema.BodyV2ColumnPrefix)
|
||||
case strings.HasPrefix(columnExpr, logstelemetryschema.BodyPromotedColumnPrefix):
|
||||
baseColumn = logstelemetryschema.BodyPromotedColumnPrefix
|
||||
fieldName = strings.TrimPrefix(columnExpr, logstelemetryschema.BodyPromotedColumnPrefix)
|
||||
}
|
||||
fieldName = strings.ReplaceAll(fieldName, "`", "")
|
||||
|
||||
@@ -228,12 +228,12 @@ func (t *telemetryMetaStore) ListJSONValues(ctx context.Context, path string, li
|
||||
}
|
||||
|
||||
if promoted {
|
||||
path = telemetrylogs.BodyPromotedColumnPrefix + path
|
||||
path = logstelemetryschema.BodyPromotedColumnPrefix + path
|
||||
} else {
|
||||
path = telemetrylogs.BodyV2ColumnPrefix + path
|
||||
path = logstelemetryschema.BodyV2ColumnPrefix + path
|
||||
}
|
||||
|
||||
from := fmt.Sprintf("%s.%s", telemetrylogs.DBName, telemetrylogs.LogsV2TableName)
|
||||
from := fmt.Sprintf("%s.%s", logstelemetryschema.DBName, logstelemetryschema.LogsV2TableName)
|
||||
colExpr := func(typ telemetrytypes.JSONDataType) string {
|
||||
return fmt.Sprintf("dynamicElement(%s, '%s')", path, typ.StringValue())
|
||||
}
|
||||
@@ -382,7 +382,7 @@ func (t *telemetryMetaStore) IsPathPromoted(ctx context.Context, path string) (b
|
||||
split := strings.Split(path, telemetrytypes.ArraySep)
|
||||
pathSegment := split[0]
|
||||
query := fmt.Sprintf("SELECT 1 FROM %s.%s WHERE signal = ? AND column_name = ? AND field_context = ? AND field_name = ? LIMIT 1", DBName, PromotedPathsTableName)
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, telemetrytypes.SignalLogs, telemetrylogs.LogsV2BodyPromotedColumn, telemetrytypes.FieldContextBody, pathSegment)
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, telemetrytypes.SignalLogs, logstelemetryschema.LogsV2BodyPromotedColumn, telemetrytypes.FieldContextBody, pathSegment)
|
||||
if err != nil {
|
||||
return false, errors.WrapInternalf(err, CodeFailCheckPathPromoted, "failed to check if path %s is promoted", path)
|
||||
}
|
||||
@@ -397,7 +397,7 @@ func (t *telemetryMetaStore) GetPromotedPaths(ctx context.Context, paths ...stri
|
||||
sb := sqlbuilder.Select("field_name").From(fmt.Sprintf("%s.%s", DBName, PromotedPathsTableName))
|
||||
conditions := []string{
|
||||
sb.Equal("signal", telemetrytypes.SignalLogs),
|
||||
sb.Equal("column_name", telemetrylogs.LogsV2BodyPromotedColumn),
|
||||
sb.Equal("column_name", logstelemetryschema.LogsV2BodyPromotedColumn),
|
||||
sb.Equal("field_context", telemetrytypes.FieldContextBody),
|
||||
sb.NotEqual("field_name", "__all__"),
|
||||
}
|
||||
@@ -433,8 +433,8 @@ func (t *telemetryMetaStore) GetPromotedPaths(ctx context.Context, paths ...stri
|
||||
// TODO(Piyush): Remove this function.
|
||||
func CleanPathPrefixes(path string) string {
|
||||
path = strings.TrimPrefix(path, telemetrytypes.BodyJSONStringSearchPrefix)
|
||||
path = strings.TrimPrefix(path, telemetrylogs.BodyV2ColumnPrefix)
|
||||
path = strings.TrimPrefix(path, telemetrylogs.BodyPromotedColumnPrefix)
|
||||
path = strings.TrimPrefix(path, logstelemetryschema.BodyV2ColumnPrefix)
|
||||
path = strings.TrimPrefix(path, logstelemetryschema.BodyPromotedColumnPrefix)
|
||||
return path
|
||||
}
|
||||
|
||||
@@ -454,7 +454,7 @@ func (t *telemetryMetaStore) PromotePaths(ctx context.Context, paths ...string)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
if err := batch.Append(telemetrytypes.SignalLogs, telemetrylogs.LogsV2BodyPromotedColumn, "JSON()", telemetrytypes.FieldContextBody, trimmed, 0, releaseTime); err != nil {
|
||||
if err := batch.Append(telemetrytypes.SignalLogs, logstelemetryschema.LogsV2BodyPromotedColumn, "JSON()", telemetrytypes.FieldContextBody, trimmed, 0, releaseTime); err != nil {
|
||||
_ = batch.Abort()
|
||||
return errors.WrapInternalf(err, CodeFailedToAppendPath, "failed to append path")
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz-otel-collector/constants"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -25,8 +25,8 @@ func TestBuildListLogsJSONIndexesQuery(t *testing.T) {
|
||||
expectedSQL: "SELECT name, type_full, expr, granularity FROM clusterAllReplicas('test-cluster', system.data_skipping_indices) " +
|
||||
"WHERE database = ? AND table = ? AND (LOWER(expr) LIKE LOWER(?) OR LOWER(expr) LIKE LOWER(?))",
|
||||
expectedArgs: []any{
|
||||
telemetrylogs.DBName,
|
||||
telemetrylogs.LogsV2LocalTableName,
|
||||
logstelemetryschema.DBName,
|
||||
logstelemetryschema.LogsV2LocalTableName,
|
||||
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyV2ColumnPrefix)),
|
||||
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyPromotedColumnPrefix)),
|
||||
},
|
||||
@@ -38,8 +38,8 @@ func TestBuildListLogsJSONIndexesQuery(t *testing.T) {
|
||||
expectedSQL: "SELECT name, type_full, expr, granularity FROM clusterAllReplicas('test-cluster', system.data_skipping_indices) " +
|
||||
"WHERE database = ? AND table = ? AND (LOWER(expr) LIKE LOWER(?) OR LOWER(expr) LIKE LOWER(?)) AND (LOWER(replaceAll(expr, '`', '')) LIKE LOWER(?) OR LOWER(replaceAll(expr, '`', '')) LIKE LOWER(?))",
|
||||
expectedArgs: []any{
|
||||
telemetrylogs.DBName,
|
||||
telemetrylogs.LogsV2LocalTableName,
|
||||
logstelemetryschema.DBName,
|
||||
logstelemetryschema.LogsV2LocalTableName,
|
||||
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyV2ColumnPrefix)),
|
||||
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyPromotedColumnPrefix)),
|
||||
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains("foo")),
|
||||
|
||||
@@ -14,11 +14,12 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryaudit"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
@@ -77,27 +78,6 @@ func escapeForLike(s string) string {
|
||||
func NewTelemetryMetaStore(
|
||||
settings factory.ProviderSettings,
|
||||
telemetrystore telemetrystore.TelemetryStore,
|
||||
tracesDBName string,
|
||||
tracesFieldsTblName string,
|
||||
spanAttributesKeysTblName string,
|
||||
indexV3TblName string,
|
||||
metricsDBName string,
|
||||
metricsFieldsTblName string,
|
||||
meterDBName string,
|
||||
meterFieldsTblName string,
|
||||
logsDBName string,
|
||||
logsV2TblName string,
|
||||
logsFieldsTblName string,
|
||||
logAttributeKeysTblName string,
|
||||
logResourceKeysTblName string,
|
||||
auditDBName string,
|
||||
auditLogsTblName string,
|
||||
auditFieldsTblName string,
|
||||
auditAttributeKeysTblName string,
|
||||
auditResourceKeysTblName string,
|
||||
relatedMetadataDBName string,
|
||||
relatedMetadataTblName string,
|
||||
columnEvolutionMetadataTblName string,
|
||||
fl flagger.Flagger,
|
||||
) telemetrytypes.MetadataStore {
|
||||
metadataSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrymetadata")
|
||||
@@ -108,32 +88,32 @@ func NewTelemetryMetaStore(
|
||||
t := &telemetryMetaStore{
|
||||
logger: metadataSettings.Logger(),
|
||||
telemetrystore: telemetrystore,
|
||||
tracesDBName: tracesDBName,
|
||||
tracesFieldsTblName: tracesFieldsTblName,
|
||||
spanAttributesKeysTblName: spanAttributesKeysTblName,
|
||||
indexV3TblName: indexV3TblName,
|
||||
metricsDBName: metricsDBName,
|
||||
metricsFieldsTblName: metricsFieldsTblName,
|
||||
meterDBName: meterDBName,
|
||||
meterFieldsTblName: meterFieldsTblName,
|
||||
logsDBName: logsDBName,
|
||||
logsV2TblName: logsV2TblName,
|
||||
logsFieldsTblName: logsFieldsTblName,
|
||||
logAttributeKeysTblName: logAttributeKeysTblName,
|
||||
logResourceKeysTblName: logResourceKeysTblName,
|
||||
auditDBName: auditDBName,
|
||||
auditLogsTblName: auditLogsTblName,
|
||||
auditFieldsTblName: auditFieldsTblName,
|
||||
auditAttributeKeysTblName: auditAttributeKeysTblName,
|
||||
auditResourceKeysTblName: auditResourceKeysTblName,
|
||||
relatedMetadataDBName: relatedMetadataDBName,
|
||||
relatedMetadataTblName: relatedMetadataTblName,
|
||||
columnEvolutionMetadataTblName: columnEvolutionMetadataTblName,
|
||||
tracesDBName: tracestelemetryschema.DBName,
|
||||
tracesFieldsTblName: tracestelemetryschema.TagAttributesV2TableName,
|
||||
spanAttributesKeysTblName: tracestelemetryschema.SpanAttributesKeysTblName,
|
||||
indexV3TblName: tracestelemetryschema.SpanIndexV3TableName,
|
||||
metricsDBName: metricstelemetryschema.DBName,
|
||||
metricsFieldsTblName: metricstelemetryschema.AttributesMetadataTableName,
|
||||
meterDBName: metertelemetryschema.DBName,
|
||||
meterFieldsTblName: metertelemetryschema.SamplesAgg1dTableName,
|
||||
logsDBName: logstelemetryschema.DBName,
|
||||
logsV2TblName: logstelemetryschema.LogsV2TableName,
|
||||
logsFieldsTblName: logstelemetryschema.TagAttributesV2TableName,
|
||||
logAttributeKeysTblName: logstelemetryschema.LogAttributeKeysTblName,
|
||||
logResourceKeysTblName: logstelemetryschema.LogResourceKeysTblName,
|
||||
auditDBName: audittelemetryschema.DBName,
|
||||
auditLogsTblName: audittelemetryschema.AuditLogsTableName,
|
||||
auditFieldsTblName: audittelemetryschema.TagAttributesTableName,
|
||||
auditAttributeKeysTblName: audittelemetryschema.LogAttributeKeysTblName,
|
||||
auditResourceKeysTblName: audittelemetryschema.LogResourceKeysTblName,
|
||||
relatedMetadataDBName: DBName,
|
||||
relatedMetadataTblName: AttributesMetadataTableName,
|
||||
columnEvolutionMetadataTblName: ColumnEvolutionMetadataTableName,
|
||||
jsonColumnMetadata: map[telemetrytypes.Signal]map[telemetrytypes.FieldContext]telemetrytypes.JSONColumnMetadata{
|
||||
telemetrytypes.SignalLogs: {
|
||||
telemetrytypes.FieldContextBody: telemetrytypes.JSONColumnMetadata{
|
||||
BaseColumn: telemetrylogs.LogsV2BodyV2Column,
|
||||
PromotedColumn: telemetrylogs.LogsV2BodyPromotedColumn,
|
||||
BaseColumn: logstelemetryschema.LogsV2BodyV2Column,
|
||||
PromotedColumn: logstelemetryschema.LogsV2BodyPromotedColumn,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -309,8 +289,8 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
|
||||
complete := rowCount <= limit
|
||||
|
||||
staticKeys := []string{"isRoot", "isEntryPoint"}
|
||||
staticKeys = append(staticKeys, maps.Keys(telemetrytraces.IntrinsicFields)...)
|
||||
staticKeys = append(staticKeys, maps.Keys(telemetrytraces.CalculatedFields)...)
|
||||
staticKeys = append(staticKeys, maps.Keys(tracestelemetryschema.IntrinsicFields)...)
|
||||
staticKeys = append(staticKeys, maps.Keys(tracestelemetryschema.CalculatedFields)...)
|
||||
|
||||
// Add matching intrinsic and matching calculated fields
|
||||
// These don't count towards the limit
|
||||
@@ -324,14 +304,14 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
|
||||
}
|
||||
|
||||
if found {
|
||||
if field, exists := telemetrytraces.IntrinsicFields[key]; exists {
|
||||
if field, exists := tracestelemetryschema.IntrinsicFields[key]; exists {
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if field, exists := telemetrytraces.CalculatedFields[key]; exists {
|
||||
if field, exists := tracestelemetryschema.CalculatedFields[key]; exists {
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
@@ -656,7 +636,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
complete := rowCount <= limit
|
||||
|
||||
staticKeys := []string{}
|
||||
staticKeys = append(staticKeys, maps.Keys(telemetrylogs.IntrinsicFields)...)
|
||||
staticKeys = append(staticKeys, maps.Keys(logstelemetryschema.IntrinsicFields)...)
|
||||
|
||||
// Add matching intrinsic and matching calculated fields
|
||||
// These don't count towards the limit
|
||||
@@ -670,7 +650,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
}
|
||||
|
||||
if found {
|
||||
if field, exists := telemetrylogs.IntrinsicFields[key]; exists {
|
||||
if field, exists := logstelemetryschema.IntrinsicFields[key]; exists {
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
@@ -898,7 +878,7 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
|
||||
complete := rowCount <= limit
|
||||
|
||||
// Add intrinsic audit fields (same as logs intrinsics: body, severity_text, etc.)
|
||||
staticKeys := maps.Keys(telemetryaudit.IntrinsicFields)
|
||||
staticKeys := maps.Keys(audittelemetryschema.IntrinsicFields)
|
||||
for _, key := range staticKeys {
|
||||
found := false
|
||||
for _, v := range searchTexts {
|
||||
@@ -909,7 +889,7 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
|
||||
}
|
||||
|
||||
if found {
|
||||
if field, exists := telemetryaudit.IntrinsicFields[key]; exists {
|
||||
if field, exists := audittelemetryschema.IntrinsicFields[key]; exists {
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
@@ -1176,7 +1156,7 @@ func enrichWithIntrinsicMetricKeys(keys map[string][]*telemetrytypes.TelemetryFi
|
||||
continue
|
||||
}
|
||||
|
||||
for name, key := range telemetrymetrics.IntrinsicMetricFieldDefinitions {
|
||||
for name, key := range metricstelemetryschema.IntrinsicMetricFieldDefinitions {
|
||||
if !selectorMatchesIntrinsicField(selector, key) {
|
||||
continue
|
||||
}
|
||||
@@ -1188,6 +1168,27 @@ func enrichWithIntrinsicMetricKeys(keys map[string][]*telemetrytypes.TelemetryFi
|
||||
return keys
|
||||
}
|
||||
|
||||
// enrichWithGenAIKeys adds keys that can be queried for GenAI signals, even though they have not been ingested yet.
|
||||
func enrichWithGenAIKeys(keys map[string][]*telemetrytypes.TelemetryFieldKey, selectors []*telemetrytypes.FieldKeySelector) map[string][]*telemetrytypes.TelemetryFieldKey {
|
||||
for _, selector := range selectors {
|
||||
if selector.Signal != telemetrytypes.SignalTraces && selector.Signal != telemetrytypes.SignalUnspecified {
|
||||
continue
|
||||
}
|
||||
for name, def := range telemetrytypes.GenAIFieldDefinitions {
|
||||
if len(keys[name]) > 0 {
|
||||
continue // already resolved from ingested data
|
||||
}
|
||||
if !selectorMatchesIntrinsicField(selector, def) {
|
||||
continue
|
||||
}
|
||||
keyCopy := def
|
||||
keys[name] = []*telemetrytypes.TelemetryFieldKey{&keyCopy}
|
||||
}
|
||||
}
|
||||
|
||||
return keys
|
||||
}
|
||||
|
||||
func selectorMatchesIntrinsicField(selector *telemetrytypes.FieldKeySelector, definition telemetrytypes.TelemetryFieldKey) bool {
|
||||
if selector.FieldContext != telemetrytypes.FieldContextUnspecified && selector.FieldContext != definition.FieldContext {
|
||||
return false
|
||||
@@ -1273,6 +1274,9 @@ func (t *telemetryMetaStore) GetKeys(ctx context.Context, orgID valuer.UUID, fie
|
||||
|
||||
applyBackwardCompatibleKeys(mapOfKeys)
|
||||
mapOfKeys = enrichWithIntrinsicMetricKeys(mapOfKeys, selectors)
|
||||
if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableAIObservability, featuretypes.NewFlaggerEvaluationContext(orgID)) {
|
||||
mapOfKeys = enrichWithGenAIKeys(mapOfKeys, selectors)
|
||||
}
|
||||
|
||||
return mapOfKeys, complete, nil
|
||||
}
|
||||
@@ -1351,6 +1355,9 @@ func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, orgID valuer.UUID
|
||||
|
||||
applyBackwardCompatibleKeys(mapOfKeys)
|
||||
mapOfKeys = enrichWithIntrinsicMetricKeys(mapOfKeys, fieldKeySelectors)
|
||||
if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableAIObservability, featuretypes.NewFlaggerEvaluationContext(orgID)) {
|
||||
mapOfKeys = enrichWithGenAIKeys(mapOfKeys, fieldKeySelectors)
|
||||
}
|
||||
|
||||
return mapOfKeys, complete, nil
|
||||
}
|
||||
@@ -1892,7 +1899,7 @@ func (t *telemetryMetaStore) getIntrinsicMetricFieldValues(ctx context.Context,
|
||||
instrumentationtypes.CodeNamespace: "metadata",
|
||||
instrumentationtypes.CodeFunctionName: "getIntrinsicMetricFieldValues",
|
||||
})
|
||||
key, ok := telemetrymetrics.IntrinsicMetricFieldDefinitions[fieldValueSelector.Name]
|
||||
key, ok := metricstelemetryschema.IntrinsicMetricFieldDefinitions[fieldValueSelector.Name]
|
||||
if !ok {
|
||||
return &telemetrytypes.TelemetryFieldValues{}, nil
|
||||
}
|
||||
@@ -1922,18 +1929,18 @@ func (t *telemetryMetaStore) getIntrinsicMetricFieldValues(ctx context.Context,
|
||||
tableName string
|
||||
extraConds []string
|
||||
}{
|
||||
{telemetrymetrics.TimeseriesV41weekTableName, nil},
|
||||
{metricstelemetryschema.TimeseriesV41weekTableName, nil},
|
||||
}
|
||||
if reductionEnabled {
|
||||
sources = append(sources,
|
||||
struct {
|
||||
tableName string
|
||||
extraConds []string
|
||||
}{telemetrymetrics.TimeseriesV4BufferTableName, []string{"is_reduced = false"}},
|
||||
}{metricstelemetryschema.TimeseriesV4BufferTableName, []string{"is_reduced = false"}},
|
||||
struct {
|
||||
tableName string
|
||||
extraConds []string
|
||||
}{telemetrymetrics.TimeseriesV4ReducedTableName, nil},
|
||||
}{metricstelemetryschema.TimeseriesV4ReducedTableName, nil},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -2262,7 +2269,7 @@ func (t *telemetryMetaStore) fetchMetricsTemporalityAndType(ctx context.Context,
|
||||
instrumentationtypes.CodeNamespace: "metadata",
|
||||
instrumentationtypes.CodeFunctionName: "fetchMetricsTemporalityAndType",
|
||||
})
|
||||
adjustedStartTs, adjustedEndTs, tsTableName, _ := telemetrymetrics.WhichTSTableToUse(queryTimeRangeStartTs, queryTimeRangeEndTs, false, nil)
|
||||
adjustedStartTs, adjustedEndTs, tsTableName, _ := metricstelemetryschema.WhichTSTableToUse(queryTimeRangeStartTs, queryTimeRangeEndTs, false, nil)
|
||||
|
||||
temporalities, types, err := t.fetchTemporalityTypeForTable(ctx, tsTableName, adjustedStartTs, adjustedEndTs, metricNames)
|
||||
if err != nil {
|
||||
@@ -2271,7 +2278,7 @@ func (t *telemetryMetaStore) fetchMetricsTemporalityAndType(ctx context.Context,
|
||||
|
||||
reduced := make(map[string]bool)
|
||||
if reductionEnabled {
|
||||
bufferTemporalities, bufferTypes, err := t.fetchTemporalityTypeForTable(ctx, telemetrymetrics.TimeseriesV4BufferTableName, adjustedStartTs, adjustedEndTs, metricNames, "is_reduced = false")
|
||||
bufferTemporalities, bufferTypes, err := t.fetchTemporalityTypeForTable(ctx, metricstelemetryschema.TimeseriesV4BufferTableName, adjustedStartTs, adjustedEndTs, metricNames, "is_reduced = false")
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
@@ -2284,7 +2291,7 @@ func (t *telemetryMetaStore) fetchMetricsTemporalityAndType(ctx context.Context,
|
||||
}
|
||||
}
|
||||
|
||||
reducedTemporalities, reducedTypes, err := t.fetchTemporalityTypeForTable(ctx, telemetrymetrics.TimeseriesV4ReducedTableName, adjustedStartTs, adjustedEndTs, metricNames)
|
||||
reducedTemporalities, reducedTypes, err := t.fetchTemporalityTypeForTable(ctx, metricstelemetryschema.TimeseriesV4ReducedTableName, adjustedStartTs, adjustedEndTs, metricNames)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
@@ -2616,13 +2623,13 @@ func (t *telemetryMetaStore) GetFirstSeenFromMetricMetadata(ctx context.Context,
|
||||
}
|
||||
|
||||
func (t *telemetryMetaStore) FetchLastSeenInfoMulti(ctx context.Context, orgID valuer.UUID, metricNames ...string) (map[string]int64, error) {
|
||||
lastSeenInfo, err := t.fetchLastSeenInfoForTable(ctx, telemetrymetrics.TimeseriesV4TableName, metricNames)
|
||||
lastSeenInfo, err := t.fetchLastSeenInfoForTable(ctx, metricstelemetryschema.TimeseriesV4TableName, metricNames)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) {
|
||||
reducedLastSeen, err := t.fetchLastSeenInfoForTable(ctx, telemetrymetrics.TimeseriesV4ReducedTableName, metricNames)
|
||||
reducedLastSeen, err := t.fetchLastSeenInfoForTable(ctx, metricstelemetryschema.TimeseriesV4ReducedTableName, metricNames)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -9,13 +9,8 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryaudit"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -42,27 +37,6 @@ func TestGetFirstSeenFromMetricMetadata(t *testing.T) {
|
||||
metadata := NewTelemetryMetaStore(
|
||||
instrumentationtest.New().ToProviderSettings(),
|
||||
mockTelemetryStore,
|
||||
telemetrytraces.DBName,
|
||||
telemetrytraces.TagAttributesV2TableName,
|
||||
telemetrytraces.SpanAttributesKeysTblName,
|
||||
telemetrytraces.SpanIndexV3TableName,
|
||||
telemetrymetrics.DBName,
|
||||
telemetrymetrics.AttributesMetadataTableName,
|
||||
telemetrymeter.DBName,
|
||||
telemetrymeter.SamplesAgg1dTableName,
|
||||
telemetrylogs.DBName,
|
||||
telemetrylogs.LogsV2TableName,
|
||||
telemetrylogs.TagAttributesV2TableName,
|
||||
telemetrylogs.LogAttributeKeysTblName,
|
||||
telemetrylogs.LogResourceKeysTblName,
|
||||
telemetryaudit.DBName,
|
||||
telemetryaudit.AuditLogsTableName,
|
||||
telemetryaudit.TagAttributesTableName,
|
||||
telemetryaudit.LogAttributeKeysTblName,
|
||||
telemetryaudit.LogResourceKeysTblName,
|
||||
DBName,
|
||||
AttributesMetadataTableName,
|
||||
ColumnEvolutionMetadataTableName,
|
||||
flaggertest.New(t),
|
||||
)
|
||||
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
package telemetrymetrics
|
||||
|
||||
import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
|
||||
var IntrinsicFields = []string{
|
||||
"temporality",
|
||||
"metric_name",
|
||||
"type",
|
||||
"is_monotonic",
|
||||
}
|
||||
|
||||
var IntrinsicMetricFieldDefinitions = map[string]telemetrytypes.TelemetryFieldKey{
|
||||
"metric_name": {
|
||||
Name: "metric_name",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextMetric,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
// "type": {
|
||||
// Name: "type",
|
||||
// Signal: telemetrytypes.SignalMetrics,
|
||||
// FieldContext: telemetrytypes.FieldContextMetric,
|
||||
// FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
// },
|
||||
// "temporality": {
|
||||
// Name: "temporality",
|
||||
// Signal: telemetrytypes.SignalMetrics,
|
||||
// FieldContext: telemetrytypes.FieldContextMetric,
|
||||
// FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
// },
|
||||
// "is_monotonic": {
|
||||
// Name: "is_monotonic",
|
||||
// Signal: telemetrytypes.SignalMetrics,
|
||||
// FieldContext: telemetrytypes.FieldContextMetric,
|
||||
// FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
// },
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetryaudit
|
||||
package audittelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetryaudit
|
||||
package audittelemetryschema
|
||||
|
||||
import (
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
@@ -127,3 +127,14 @@ var auditLogColumns = map[string]*schema.Column{
|
||||
"scope_version": {Name: "scope_version", Type: schema.ColumnTypeString},
|
||||
"scope_string": {Name: "scope_string", Type: schema.MapColumnType{KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, ValueType: schema.ColumnTypeString}},
|
||||
}
|
||||
|
||||
const (
|
||||
DBName = "signoz_audit"
|
||||
AuditLogsTableName = "distributed_logs"
|
||||
AuditLogsLocalTableName = "logs"
|
||||
TagAttributesTableName = "distributed_tag_attributes"
|
||||
TagAttributesLocalTableName = "tag_attributes"
|
||||
LogAttributeKeysTblName = "distributed_logs_attribute_keys"
|
||||
LogResourceKeysTblName = "distributed_logs_resource_keys"
|
||||
LogsResourceTableName = "distributed_logs_resource"
|
||||
)
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetryaudit
|
||||
package audittelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrylogs
|
||||
package logstelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrylogs
|
||||
package logstelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -548,7 +548,7 @@ func TestConditionForSynthesizedPrefixedKeys(t *testing.T) {
|
||||
t.Run("bare intrinsic column resolves to the column, not synthesized attributes", func(t *testing.T) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
key := telemetrytypes.TelemetryFieldKey{Name: "severity_text"}
|
||||
conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "ERROR", sb)
|
||||
conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "ERROR", sb)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, conds, 1)
|
||||
sb.Where(conds...)
|
||||
@@ -560,7 +560,7 @@ func TestConditionForSynthesizedPrefixedKeys(t *testing.T) {
|
||||
t.Run("attribute context", func(t *testing.T) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
key := telemetrytypes.TelemetryFieldKey{Name: "custom.key", FieldContext: telemetrytypes.FieldContextAttribute}
|
||||
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "v", sb)
|
||||
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "v", sb)
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, warnings)
|
||||
require.Len(t, conds, 2)
|
||||
@@ -571,7 +571,7 @@ func TestConditionForSynthesizedPrefixedKeys(t *testing.T) {
|
||||
t.Run("resource context", func(t *testing.T) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
key := telemetrytypes.TelemetryFieldKey{Name: "custom.key", FieldContext: telemetrytypes.FieldContextResource}
|
||||
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "v", sb)
|
||||
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "v", sb)
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, warnings)
|
||||
require.Len(t, conds, 2)
|
||||
@@ -582,7 +582,7 @@ func TestConditionForSynthesizedPrefixedKeys(t *testing.T) {
|
||||
t.Run("log context folds to attributes then body", func(t *testing.T) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
key := telemetrytypes.TelemetryFieldKey{Name: "custom.key", FieldContext: telemetrytypes.FieldContextLog}
|
||||
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "v", sb)
|
||||
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "v", sb)
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, warnings)
|
||||
require.Len(t, conds, 4)
|
||||
@@ -1,8 +1,6 @@
|
||||
package telemetrylogs
|
||||
package logstelemetryschema
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/SigNoz/signoz-otel-collector/constants"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -41,9 +39,8 @@ const (
|
||||
|
||||
// messageSubColumn is the ClickHouse sub-column that body searches map to
|
||||
// when use_json_body feature flag is true.
|
||||
messageSubField = "message"
|
||||
messageSubColumn = "body_v2.message"
|
||||
bodySearchDefaultWarning = "body searches default to `body.message:string`. Use `body.<key>` to search a different field inside body"
|
||||
messageSubField = "message"
|
||||
messageSubColumn = "body_v2.message"
|
||||
|
||||
// bodyMessageField is the field name addressing the message sub-field of the
|
||||
// body when use_json_body is enabled (i.e. `body` + `.` + `message`). hasToken
|
||||
@@ -137,10 +134,16 @@ var (
|
||||
}
|
||||
)
|
||||
|
||||
func bodyAliasExpression(bodyJSONEnabled bool) string {
|
||||
if !bodyJSONEnabled {
|
||||
return LogsV2BodyColumn
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s as body", LogsV2BodyV2Column)
|
||||
}
|
||||
const (
|
||||
DBName = "signoz_logs"
|
||||
LogsV2TableName = "distributed_logs_v2"
|
||||
LogsV2LocalTableName = "logs_v2"
|
||||
TagAttributesV2TableName = "distributed_tag_attributes_v2"
|
||||
TagAttributesV2LocalTableName = "tag_attributes_v2"
|
||||
LogAttributeKeysTblName = "distributed_logs_attribute_keys"
|
||||
LogResourceKeysTblName = "distributed_logs_resource_keys"
|
||||
LogsResourceV2TableName = "distributed_logs_v2_resource"
|
||||
PathTypesTableName = "distributed_json_path_types"
|
||||
PromotedPathsTableName = "distributed_json_promoted_paths"
|
||||
SkipIndexTableName = "system.data_skipping_indices"
|
||||
)
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrylogs
|
||||
package logstelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrylogs
|
||||
package logstelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrylogs
|
||||
package logstelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -19,7 +19,7 @@ func TestLikeAndILikeWithoutWildcards_Warns(t *testing.T) {
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
keys := BuildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
@@ -57,7 +57,7 @@ func TestLikeAndILikeWithWildcards_NoWarn(t *testing.T) {
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
keys := BuildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: context.Background(),
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrylogs
|
||||
package logstelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -21,7 +21,7 @@ func TestFilterExprLogsBodyJSON(t *testing.T) {
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
// Define a comprehensive set of field keys to support all test cases
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
keys := BuildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: context.Background(),
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrylogs
|
||||
package logstelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -41,7 +41,7 @@ func TestFilterExprLogs(t *testing.T) {
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
// Define a comprehensive set of field keys to support all test cases
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
keys := BuildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
@@ -2476,7 +2476,7 @@ func TestFilterExprLogsConflictNegation(t *testing.T) {
|
||||
|
||||
// Define a comprehensive set of field keys to support all test cases
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
keys := BuildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
keys["body"] = []*telemetrytypes.TelemetryFieldKey{
|
||||
{
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrylogs
|
||||
package logstelemetryschema
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrylogs
|
||||
package logstelemetryschema
|
||||
|
||||
import (
|
||||
"strings"
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrylogs
|
||||
package logstelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrylogs
|
||||
package logstelemetryschema
|
||||
|
||||
import (
|
||||
"testing"
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrylogs
|
||||
package logstelemetryschema
|
||||
|
||||
import (
|
||||
"strings"
|
||||
@@ -19,7 +19,7 @@ func limitString(s string, maxLen int) string {
|
||||
}
|
||||
|
||||
// Function to build a complete field key map for testing all scenarios.
|
||||
func buildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytypes.TelemetryFieldKey {
|
||||
func BuildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytypes.TelemetryFieldKey {
|
||||
keysMap := map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
"service.name": {
|
||||
{
|
||||
@@ -953,7 +953,7 @@ func buildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytype
|
||||
return keysMap
|
||||
}
|
||||
|
||||
func buildCompleteFieldKeyMapCollision() map[string][]*telemetrytypes.TelemetryFieldKey {
|
||||
func BuildCompleteFieldKeyMapCollision() map[string][]*telemetrytypes.TelemetryFieldKey {
|
||||
keysMap := map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
"service.name": {
|
||||
{
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrymeter
|
||||
package metertelemetryschema
|
||||
|
||||
import (
|
||||
"time"
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrymetrics
|
||||
package metricstelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrymetrics
|
||||
package metricstelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrymetrics
|
||||
package metricstelemetryschema
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -46,10 +47,10 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
oneHourInMilliseconds = uint64(time.Hour.Milliseconds() * 1)
|
||||
OneHourInMilliseconds = uint64(time.Hour.Milliseconds() * 1)
|
||||
sixHoursInMilliseconds = uint64(time.Hour.Milliseconds() * 6)
|
||||
oneDayInMilliseconds = uint64(time.Hour.Milliseconds() * 24)
|
||||
oneWeekInMilliseconds = uint64(oneDayInMilliseconds * 7)
|
||||
OneDayInMilliseconds = uint64(time.Hour.Milliseconds() * 24)
|
||||
oneWeekInMilliseconds = uint64(OneDayInMilliseconds * 7)
|
||||
|
||||
// when the query requests for almost 1 day, but not exactly 1 day, we need to add an offset to the end time
|
||||
// to make sure that we are using the correct table
|
||||
@@ -68,7 +69,7 @@ func WhichTSTableToUse(
|
||||
// the buffer holds the recent raw window for reduced metrics and has the same
|
||||
// shape as time_series_v4; round the start to the hour like the v4 table.
|
||||
if useBuffer {
|
||||
start = start - (start % (oneHourInMilliseconds))
|
||||
start = start - (start % (OneHourInMilliseconds))
|
||||
return start, end, TimeseriesV4BufferTableName, TimeseriesV4BufferLocalTableName
|
||||
}
|
||||
|
||||
@@ -80,7 +81,7 @@ func WhichTSTableToUse(
|
||||
switch tableHints.TimeSeriesTableName {
|
||||
case TimeseriesV4LocalTableName:
|
||||
// adjust the start time to nearest 1 hour
|
||||
start = start - (start % (oneHourInMilliseconds))
|
||||
start = start - (start % (OneHourInMilliseconds))
|
||||
distributedTableName = TimeseriesV4TableName
|
||||
case TimeseriesV46hrsLocalTableName:
|
||||
// adjust the start time to nearest 6 hours
|
||||
@@ -88,7 +89,7 @@ func WhichTSTableToUse(
|
||||
distributedTableName = TimeseriesV46hrsTableName
|
||||
case TimeseriesV41dayLocalTableName:
|
||||
// adjust the start time to nearest 1 day
|
||||
start = start - (start % (oneDayInMilliseconds))
|
||||
start = start - (start % (OneDayInMilliseconds))
|
||||
distributedTableName = TimeseriesV41dayTableName
|
||||
case TimeseriesV41weekLocalTableName:
|
||||
// adjust the start time to nearest 1 week
|
||||
@@ -107,17 +108,17 @@ func WhichTSTableToUse(
|
||||
var localTableName string
|
||||
if end-start < sixHoursInMilliseconds {
|
||||
// adjust the start time to nearest 1 hour
|
||||
start = start - (start % (oneHourInMilliseconds))
|
||||
start = start - (start % (OneHourInMilliseconds))
|
||||
distributedTableName = TimeseriesV4TableName
|
||||
localTableName = TimeseriesV4LocalTableName
|
||||
} else if end-start < oneDayInMilliseconds {
|
||||
} else if end-start < OneDayInMilliseconds {
|
||||
// adjust the start time to nearest 6 hours
|
||||
start = start - (start % (sixHoursInMilliseconds))
|
||||
distributedTableName = TimeseriesV46hrsTableName
|
||||
localTableName = TimeseriesV46hrsLocalTableName
|
||||
} else if end-start < oneWeekInMilliseconds {
|
||||
// adjust the start time to nearest 1 day
|
||||
start = start - (start % (oneDayInMilliseconds))
|
||||
start = start - (start % (OneDayInMilliseconds))
|
||||
distributedTableName = TimeseriesV41dayTableName
|
||||
localTableName = TimeseriesV41dayLocalTableName
|
||||
} else {
|
||||
@@ -206,7 +207,7 @@ func WhichSamplesTableToUse(
|
||||
return SamplesV4TableName, SamplesV4LocalTableName
|
||||
}
|
||||
|
||||
if end-start < oneDayInMilliseconds+offsetBucket {
|
||||
if end-start < OneDayInMilliseconds+offsetBucket {
|
||||
return SamplesV4TableName, SamplesV4LocalTableName
|
||||
} else if end-start < oneWeekInMilliseconds+offsetBucket {
|
||||
return SamplesV4Agg5mTableName, SamplesV4Agg5mLocalTableName
|
||||
@@ -430,3 +431,37 @@ func AggregationQueryForHistogramCountWithParams(param *metrictypes.ComparisonSp
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
var IntrinsicFields = []string{
|
||||
"temporality",
|
||||
"metric_name",
|
||||
"type",
|
||||
"is_monotonic",
|
||||
}
|
||||
|
||||
var IntrinsicMetricFieldDefinitions = map[string]telemetrytypes.TelemetryFieldKey{
|
||||
"metric_name": {
|
||||
Name: "metric_name",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextMetric,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
// "type": {
|
||||
// Name: "type",
|
||||
// Signal: telemetrytypes.SignalMetrics,
|
||||
// FieldContext: telemetrytypes.FieldContextMetric,
|
||||
// FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
// },
|
||||
// "temporality": {
|
||||
// Name: "temporality",
|
||||
// Signal: telemetrytypes.SignalMetrics,
|
||||
// FieldContext: telemetrytypes.FieldContextMetric,
|
||||
// FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
// },
|
||||
// "is_monotonic": {
|
||||
// Name: "is_monotonic",
|
||||
// Signal: telemetrytypes.SignalMetrics,
|
||||
// FieldContext: telemetrytypes.FieldContextMetric,
|
||||
// FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
// },
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package telemetrymetrics
|
||||
package metricstelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user