mirror of
https://github.com/SigNoz/signoz.git
synced 2026-07-29 09:10:40 +01:00
Compare commits
3 Commits
refactor/q
...
v2-read-pa
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
04be7222d7 | ||
|
|
bca2370862 | ||
|
|
591837152e |
1
.github/workflows/integrationci.yaml
vendored
1
.github/workflows/integrationci.yaml
vendored
@@ -54,6 +54,7 @@ jobs:
|
||||
- querierscalar
|
||||
- queriercommon
|
||||
- rawexportdata
|
||||
- promqlconformance
|
||||
- querierauthz
|
||||
- role
|
||||
- rootuser
|
||||
|
||||
@@ -2,9 +2,9 @@ package main
|
||||
|
||||
import (
|
||||
"github.com/SigNoz/signoz/pkg/metercollector"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
)
|
||||
@@ -25,8 +25,8 @@ var meterConfigs = []metercollector.Config{
|
||||
Name: zeustypes.MeterLogSize,
|
||||
Unit: zeustypes.MeterUnitBytes,
|
||||
Aggregation: zeustypes.MeterAggregationSum,
|
||||
DBName: logstelemetryschema.DBName,
|
||||
TableName: logstelemetryschema.LogsV2LocalTableName,
|
||||
DBName: telemetrylogs.DBName,
|
||||
TableName: telemetrylogs.LogsV2LocalTableName,
|
||||
DefaultRetentionDays: retentiontypes.DefaultLogsRetentionDays,
|
||||
},
|
||||
},
|
||||
@@ -36,8 +36,8 @@ var meterConfigs = []metercollector.Config{
|
||||
Name: zeustypes.MeterSpanSize,
|
||||
Unit: zeustypes.MeterUnitBytes,
|
||||
Aggregation: zeustypes.MeterAggregationSum,
|
||||
DBName: tracestelemetryschema.DBName,
|
||||
TableName: tracestelemetryschema.SpanIndexV3LocalTableName,
|
||||
DBName: telemetrytraces.DBName,
|
||||
TableName: telemetrytraces.SpanIndexV3LocalTableName,
|
||||
DefaultRetentionDays: retentiontypes.DefaultTracesRetentionDays,
|
||||
},
|
||||
},
|
||||
@@ -47,8 +47,8 @@ var meterConfigs = []metercollector.Config{
|
||||
Name: zeustypes.MeterDatapointCount,
|
||||
Unit: zeustypes.MeterUnitCount,
|
||||
Aggregation: zeustypes.MeterAggregationSum,
|
||||
DBName: metricstelemetryschema.DBName,
|
||||
TableName: metricstelemetryschema.SamplesV4LocalTableName,
|
||||
DBName: telemetrymetrics.DBName,
|
||||
TableName: telemetrymetrics.SamplesV4LocalTableName,
|
||||
DefaultRetentionDays: retentiontypes.DefaultMetricsRetentionDays,
|
||||
},
|
||||
},
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/metercollector"
|
||||
"github.com/SigNoz/signoz/pkg/modules/retention"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/licensetypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
@@ -153,7 +153,7 @@ func (provider *Provider) Collect(
|
||||
func buildOriginQuery(meterName string) (string, []any) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("toInt64(ifNull(min(unix_milli), 0))")
|
||||
sb.From(metertelemetryschema.DBName + "." + metertelemetryschema.SamplesTableName)
|
||||
sb.From(telemetrymeter.DBName + "." + telemetrymeter.SamplesTableName)
|
||||
sb.Where(sb.Equal("metric_name", meterName))
|
||||
return sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
}
|
||||
@@ -171,7 +171,7 @@ func buildQuery(meterName string, segment *retentiontypes.RetentionPolicySegment
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(selects...)
|
||||
sb.From(metertelemetryschema.DBName + "." + metertelemetryschema.SamplesTableName)
|
||||
sb.From(telemetrymeter.DBName + "." + telemetrymeter.SamplesTableName)
|
||||
sb.Where(
|
||||
sb.Equal("metric_name", meterName),
|
||||
sb.GTE("unix_milli", segment.StartMs),
|
||||
|
||||
@@ -8,16 +8,16 @@ import (
|
||||
sqlbuilder "github.com/huandu/go-sqlbuilder"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/metricreductionruletypes"
|
||||
)
|
||||
|
||||
var (
|
||||
reductionRulesTable = metricstelemetryschema.DBName + "." + metricstelemetryschema.ReductionRulesTableName
|
||||
metadataTable = metricstelemetryschema.DBName + "." + metricstelemetryschema.AttributesMetadataTableName
|
||||
bufferSeriesTable = metricstelemetryschema.DBName + "." + metricstelemetryschema.TimeseriesV4BufferTableName
|
||||
reductionRulesTable = telemetrymetrics.DBName + "." + telemetrymetrics.ReductionRulesTableName
|
||||
metadataTable = telemetrymetrics.DBName + "." + telemetrymetrics.AttributesMetadataTableName
|
||||
bufferSeriesTable = telemetrymetrics.DBName + "." + telemetrymetrics.TimeseriesV4BufferTableName
|
||||
)
|
||||
|
||||
const timeSeriesBucketMilli = int64(time.Hour / time.Millisecond)
|
||||
@@ -192,7 +192,7 @@ func (c *clickhouse) ingestedSeriesCount(ctx context.Context, metricNames []stri
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("metric_name", "uniq(fingerprint)")
|
||||
sb.From(metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4BufferTableName)
|
||||
sb.From(telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName)
|
||||
conds := []string{
|
||||
sb.In("metric_name", names...),
|
||||
sb.GE("unix_milli", startMs),
|
||||
@@ -229,8 +229,8 @@ func (c *clickhouse) ingestedSeriesCount(ctx context.Context, metricNames []stri
|
||||
// reduced sample tables.
|
||||
func (c *clickhouse) reducedSeriesCount(ctx context.Context, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (map[string]uint64, error) {
|
||||
out := make(map[string]uint64, len(metricNames))
|
||||
for _, table := range []string{metricstelemetryschema.SamplesV4ReducedLastTableName, metricstelemetryschema.SamplesV4ReducedSumTableName} {
|
||||
counts, err := c.reducedSeriesCountForTable(ctx, metricstelemetryschema.DBName+"."+table, metricNames, effectiveFrom, startMs, endMs)
|
||||
for _, table := range []string{telemetrymetrics.SamplesV4ReducedLastTableName, telemetrymetrics.SamplesV4ReducedSumTableName} {
|
||||
counts, err := c.reducedSeriesCountForTable(ctx, telemetrymetrics.DBName+"."+table, metricNames, effectiveFrom, startMs, endMs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -300,9 +300,9 @@ func (c *clickhouse) RankByVolume(ctx context.Context, metricNames []string, eff
|
||||
direction = "DESC"
|
||||
}
|
||||
|
||||
ingestedTable := metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4BufferTableName
|
||||
reducedLast := metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4ReducedLastTableName
|
||||
reducedSum := metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4ReducedSumTableName
|
||||
ingestedTable := telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName
|
||||
reducedLast := telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4ReducedLastTableName
|
||||
reducedSum := telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4ReducedSumTableName
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("base.metric_name AS metric_name", "ifNull(i.cnt, 0) AS ingested", "ifNull(d.cnt, 0) AS reduced")
|
||||
@@ -352,16 +352,16 @@ func (c *clickhouse) SampleVolumeByMetric(ctx context.Context, metricNames []str
|
||||
}
|
||||
ctx = c.withThreads(ctx)
|
||||
|
||||
ingested, err := c.countSamplesByMetric(ctx, metricstelemetryschema.DBName+"."+metricstelemetryschema.SamplesV4BufferTableName, "count()", metricNames, effectiveFrom, startMs, endMs)
|
||||
ingested, err := c.countSamplesByMetric(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4BufferTableName, "count()", metricNames, effectiveFrom, startMs, endMs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
last, err := c.countSamplesByMetric(ctx, metricstelemetryschema.DBName+"."+metricstelemetryschema.SamplesV4ReducedLastTableName, "uniq(reduced_fingerprint, unix_milli)", metricNames, effectiveFrom, startMs, endMs)
|
||||
last, err := c.countSamplesByMetric(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4ReducedLastTableName, "uniq(reduced_fingerprint, unix_milli)", metricNames, effectiveFrom, startMs, endMs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sum, err := c.countSamplesByMetric(ctx, metricstelemetryschema.DBName+"."+metricstelemetryschema.SamplesV4ReducedSumTableName, "uniq(reduced_fingerprint, unix_milli)", metricNames, effectiveFrom, startMs, endMs)
|
||||
sum, err := c.countSamplesByMetric(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4ReducedSumTableName, "uniq(reduced_fingerprint, unix_milli)", metricNames, effectiveFrom, startMs, endMs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -419,7 +419,7 @@ func (c *clickhouse) TotalVolume(ctx context.Context, startMs, endMs int64) (uin
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("uniq(fingerprint)", "count()")
|
||||
sb.From(metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4BufferTableName)
|
||||
sb.From(telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName)
|
||||
sb.Where(sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs))
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
@@ -470,7 +470,7 @@ func (c *clickhouse) SampleTimeseries(ctx context.Context, ruledMetrics []string
|
||||
func (c *clickhouse) totalSamplesByBucket(ctx context.Context, startMs, endMs int64) (map[int64]uint64, error) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(sampleBucketExpr, "count()")
|
||||
sb.From(metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4BufferTableName)
|
||||
sb.From(telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName)
|
||||
sb.Where(sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs))
|
||||
sb.GroupBy("bucket")
|
||||
|
||||
@@ -485,7 +485,7 @@ func (c *clickhouse) ruledIngestedSamplesByBucket(ctx context.Context, metricNam
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(sampleBucketExpr, "count()")
|
||||
sb.From(metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4BufferTableName)
|
||||
sb.From(telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName)
|
||||
conds := []string{sb.In("metric_name", names...), sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs)}
|
||||
if len(effectiveFrom) > 0 {
|
||||
conds = append(conds, strictEffectiveFrom(sb, metricNames, effectiveFrom))
|
||||
@@ -499,7 +499,7 @@ func (c *clickhouse) ruledIngestedSamplesByBucket(ctx context.Context, metricNam
|
||||
// reduced 60s rows are versioned by computed_at, so count distinct buckets.
|
||||
func (c *clickhouse) ruledRetainedSamplesByBucket(ctx context.Context, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (map[int64]uint64, error) {
|
||||
out := make(map[int64]uint64)
|
||||
for _, table := range []string{metricstelemetryschema.SamplesV4ReducedLastTableName, metricstelemetryschema.SamplesV4ReducedSumTableName} {
|
||||
for _, table := range []string{telemetrymetrics.SamplesV4ReducedLastTableName, telemetrymetrics.SamplesV4ReducedSumTableName} {
|
||||
names := make([]any, len(metricNames))
|
||||
for i, name := range metricNames {
|
||||
names[i] = name
|
||||
@@ -507,7 +507,7 @@ func (c *clickhouse) ruledRetainedSamplesByBucket(ctx context.Context, metricNam
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(sampleBucketExpr, "uniq(reduced_fingerprint, unix_milli)")
|
||||
sb.From(metricstelemetryschema.DBName + "." + table)
|
||||
sb.From(telemetrymetrics.DBName + "." + table)
|
||||
conds := []string{sb.In("metric_name", names...), sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs)}
|
||||
if len(effectiveFrom) > 0 {
|
||||
conds = append(conds, strictEffectiveFrom(sb, metricNames, effectiveFrom))
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
.warning-popover-overlay {
|
||||
--antd-arrow-background-color: var(--l2-background);
|
||||
}
|
||||
|
||||
.warning-content {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
background-color: var(--l2-background);
|
||||
|
||||
// === SECTION: Summary (Top)
|
||||
&__summary-section {
|
||||
@@ -10,16 +15,25 @@
|
||||
|
||||
&__summary {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
gap: var(--spacing-8);
|
||||
padding: 16px;
|
||||
}
|
||||
|
||||
&__summary-left {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
align-items: flex-start;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
&__icon-wrapper {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex-shrink: 0;
|
||||
height: var(--spacing-12);
|
||||
}
|
||||
|
||||
&__summary-text {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
|
||||
@@ -2,6 +2,7 @@ import { ReactNode, useMemo } from 'react';
|
||||
import { Color } from '@signozhq/design-tokens';
|
||||
import { Button, Popover, PopoverProps } from 'antd';
|
||||
import ErrorIcon from 'assets/Error';
|
||||
import cx from 'classnames';
|
||||
import OverlayScrollbar from 'components/OverlayScrollbar/OverlayScrollbar';
|
||||
import { BookOpenText, ChevronsDown, TriangleAlert } from '@signozhq/icons';
|
||||
import KeyValueLabel from 'periscope/components/KeyValueLabel';
|
||||
@@ -30,16 +31,18 @@ export function WarningContent({ warning }: WarningContentProps): JSX.Element {
|
||||
{/* Summary Header */}
|
||||
<section className="warning-content__summary-section">
|
||||
<header className="warning-content__summary">
|
||||
<div className="warning-content__summary-left">
|
||||
<div className="warning-content__icon-wrapper">
|
||||
<ErrorIcon />
|
||||
</div>
|
||||
{(warningCode || warningMessage) && (
|
||||
<div className="warning-content__summary-left">
|
||||
<div className="warning-content__icon-wrapper">
|
||||
<ErrorIcon />
|
||||
</div>
|
||||
|
||||
<div className="warning-content__summary-text">
|
||||
<h2 className="warning-content__warning-code">{warningCode}</h2>
|
||||
<p className="warning-content__warning-message">{warningMessage}</p>
|
||||
<div className="warning-content__summary-text">
|
||||
<h2 className="warning-content__warning-code">{warningCode}</h2>
|
||||
<p className="warning-content__warning-message">{warningMessage}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{warningUrl && (
|
||||
<div className="warning-content__summary-right">
|
||||
@@ -154,6 +157,10 @@ function WarningPopover({
|
||||
overlayInnerStyle={{ padding: 0 }}
|
||||
autoAdjustOverflow
|
||||
{...popoverProps}
|
||||
overlayClassName={cx(
|
||||
'warning-popover-overlay',
|
||||
popoverProps.overlayClassName,
|
||||
)}
|
||||
>
|
||||
{children || (
|
||||
<TriangleAlert
|
||||
|
||||
@@ -5,14 +5,14 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
func (m *module) Collect(ctx context.Context, _ valuer.UUID) (map[string]any, error) {
|
||||
stats := make(map[string]any)
|
||||
|
||||
metadataTable := fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName)
|
||||
metadataTable := fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName)
|
||||
var (
|
||||
systemMetricCount uint64
|
||||
k8sMetricCount uint64
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -281,7 +281,7 @@ func (m *module) getPerGroupContainerStatusCounts(
|
||||
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
|
||||
|
||||
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
|
||||
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
|
||||
// Built once; identical across the two fps CTEs (buildFilterClause hits the
|
||||
// metadata store + parses the expression). AddWhereClause only reads it.
|
||||
@@ -310,7 +310,7 @@ func (m *module) getPerGroupContainerStatusCounts(
|
||||
)
|
||||
}
|
||||
stateFps.Select(stateFpsCols...)
|
||||
stateFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
stateFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
stateFps.Where(
|
||||
stateFps.E("metric_name", containerStatusStateMetricName),
|
||||
stateFps.GE("unix_milli", tsAdjustedStart),
|
||||
@@ -342,7 +342,7 @@ func (m *module) getPerGroupContainerStatusCounts(
|
||||
containerState.Select(containerStateCols...)
|
||||
containerState.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN state_fps AS fps ON samples.fingerprint = fps.fingerprint",
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
))
|
||||
containerState.Where(
|
||||
containerState.E("samples.metric_name", containerStatusStateMetricName),
|
||||
@@ -361,7 +361,7 @@ func (m *module) getPerGroupContainerStatusCounts(
|
||||
fmt.Sprintf("JSONExtractString(labels, %s) AS container_name", reasonFps.Var(containerNameAttrKey)),
|
||||
fmt.Sprintf("JSONExtractString(labels, %s) AS reason", reasonFps.Var(containerStatusReasonAttrKey)),
|
||||
)
|
||||
reasonFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
reasonFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
reasonFps.Where(
|
||||
reasonFps.E("metric_name", containerStatusReasonMetricName),
|
||||
reasonFps.GE("unix_milli", tsAdjustedStart),
|
||||
@@ -391,7 +391,7 @@ func (m *module) getPerGroupContainerStatusCounts(
|
||||
)
|
||||
reasonInner.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN reason_fps AS fps ON samples.fingerprint = fps.fingerprint",
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
))
|
||||
reasonInner.Where(
|
||||
reasonInner.E("samples.metric_name", containerStatusReasonMetricName),
|
||||
@@ -546,7 +546,7 @@ func (m *module) getPerGroupContainerRestartCounts(
|
||||
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
|
||||
|
||||
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
|
||||
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
|
||||
var (
|
||||
filterClause *sqlbuilder.WhereClause
|
||||
@@ -572,7 +572,7 @@ func (m *module) getPerGroupContainerRestartCounts(
|
||||
)
|
||||
}
|
||||
restartFps.Select(restartFpsCols...)
|
||||
restartFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
restartFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
restartFps.Where(
|
||||
restartFps.E("metric_name", containerRestartsMetricName),
|
||||
restartFps.GE("unix_milli", tsAdjustedStart),
|
||||
@@ -602,7 +602,7 @@ func (m *module) getPerGroupContainerRestartCounts(
|
||||
containerRestarts.Select(containerRestartsCols...)
|
||||
containerRestarts.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN restart_fps AS fps ON samples.fingerprint = fps.fingerprint",
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
))
|
||||
containerRestarts.Where(
|
||||
containerRestarts.E("samples.metric_name", containerRestartsMetricName),
|
||||
@@ -690,7 +690,7 @@ func (m *module) getPerGroupContainerReadyCounts(
|
||||
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
|
||||
|
||||
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
|
||||
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
|
||||
var (
|
||||
filterClause *sqlbuilder.WhereClause
|
||||
@@ -716,7 +716,7 @@ func (m *module) getPerGroupContainerReadyCounts(
|
||||
)
|
||||
}
|
||||
readyFps.Select(readyFpsCols...)
|
||||
readyFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
readyFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
readyFps.Where(
|
||||
readyFps.E("metric_name", containerReadyMetricName),
|
||||
readyFps.GE("unix_milli", tsAdjustedStart),
|
||||
@@ -746,7 +746,7 @@ func (m *module) getPerGroupContainerReadyCounts(
|
||||
containerReady.Select(containerReadyCols...)
|
||||
containerReady.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN ready_fps AS fps ON samples.fingerprint = fps.fingerprint",
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
))
|
||||
containerReady.Where(
|
||||
containerReady.E("samples.metric_name", containerReadyMetricName),
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
@@ -344,11 +344,11 @@ func alignedMetricWindow(startMs, endMs int64) (
|
||||
flooredEndMs = flooredEndMs - (flooredEndMs % (adjustStep * 1000))
|
||||
}
|
||||
|
||||
tsAdjustedStartMs, _, distributedTSTable, localTSTable := metricstelemetryschema.WhichTSTableToUse(
|
||||
tsAdjustedStartMs, _, distributedTSTable, localTSTable := telemetrymetrics.WhichTSTableToUse(
|
||||
samplesAdjustedStartMs, flooredEndMs, false, nil,
|
||||
)
|
||||
|
||||
distributedSamplesTable, localSamplesTable := metricstelemetryschema.WhichSamplesTableToUse(
|
||||
distributedSamplesTable, localSamplesTable := telemetrymetrics.WhichSamplesTableToUse(
|
||||
samplesAdjustedStartMs, flooredEndMs,
|
||||
metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil,
|
||||
)
|
||||
@@ -362,7 +362,7 @@ func alignedMetricWindow(startMs, endMs int64) (
|
||||
func (m *module) buildSamplesTblFingerprintSubQuery(metricNames []string, samplesTable string, flooredStart, flooredEnd uint64) *sqlbuilder.SelectBuilder {
|
||||
fpSB := sqlbuilder.NewSelectBuilder()
|
||||
fpSB.Select("DISTINCT fingerprint")
|
||||
fpSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, samplesTable))
|
||||
fpSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, samplesTable))
|
||||
fpSB.Where(
|
||||
fpSB.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
fpSB.GE("unix_milli", flooredStart),
|
||||
@@ -376,7 +376,7 @@ func (m *module) buildSamplesTblFingerprintSubQuery(metricNames []string, sample
|
||||
func (m *module) buildReducedSamplesTblFingerprintSubQuery(metricNames []string, flooredStart, flooredEnd uint64) *sqlbuilder.SelectBuilder {
|
||||
lastSB := sqlbuilder.NewSelectBuilder()
|
||||
lastSB.Select("reduced_fingerprint")
|
||||
lastSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedLastTableName))
|
||||
lastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName))
|
||||
lastSB.Where(
|
||||
lastSB.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
lastSB.GE("unix_milli", flooredStart),
|
||||
@@ -385,7 +385,7 @@ func (m *module) buildReducedSamplesTblFingerprintSubQuery(metricNames []string,
|
||||
|
||||
sumSB := sqlbuilder.NewSelectBuilder()
|
||||
sumSB.Select("reduced_fingerprint")
|
||||
sumSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedSumTableName))
|
||||
sumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName))
|
||||
sumSB.Where(
|
||||
sumSB.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
sumSB.GE("unix_milli", flooredStart),
|
||||
@@ -478,7 +478,7 @@ func (m *module) getEarliestMetricTime(ctx context.Context, metricNames []string
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("min(first_reported_unix_milli) AS min_first_reported")
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
|
||||
sb.Where(sb.In("metric_name", sqlbuilder.List(metricNames)))
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
@@ -504,7 +504,7 @@ func (m *module) getMetricsExistence(ctx context.Context, metricNames []string)
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("metric_name", "count(*) AS cnt")
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
|
||||
sb.Where(sb.In("metric_name", sqlbuilder.List(metricNames)))
|
||||
sb.GroupBy("metric_name")
|
||||
|
||||
@@ -549,7 +549,7 @@ func (m *module) getAttributesExistence(ctx context.Context, metricNames, attrNa
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("attr_name", "count(*) AS cnt")
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
|
||||
sb.Where(
|
||||
sb.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
sb.In("attr_name", sqlbuilder.List(attrNames)),
|
||||
@@ -656,7 +656,7 @@ func (m *module) getMetadata(
|
||||
|
||||
rawSrc := sqlbuilder.NewSelectBuilder()
|
||||
rawSrc.Select("labels", "unix_milli")
|
||||
rawSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTableName))
|
||||
rawSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTableName))
|
||||
rawSrc.Where(
|
||||
rawSrc.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
rawSrc.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -669,7 +669,7 @@ func (m *module) getMetadata(
|
||||
|
||||
reducedSrc := sqlbuilder.NewSelectBuilder()
|
||||
reducedSrc.Select("labels", "unix_milli")
|
||||
reducedSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
|
||||
reducedSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
|
||||
reducedSrc.Where(
|
||||
reducedSrc.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
reducedSrc.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -683,7 +683,7 @@ func (m *module) getMetadata(
|
||||
// Inner query reads over the union of raw + reduced series.
|
||||
innerSB.From(innerSB.BuilderAs(sqlbuilder.UnionAll(rawSrc, reducedSrc), "series"))
|
||||
} else {
|
||||
innerSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTableName))
|
||||
innerSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTableName))
|
||||
innerSB.Where(
|
||||
innerSB.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
innerSB.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -870,7 +870,7 @@ func (m *module) getPerGroupDistinctCounts(
|
||||
|
||||
rawSrc := sqlbuilder.NewSelectBuilder()
|
||||
rawSrc.Select("labels")
|
||||
rawSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTimeSeriesTbl))
|
||||
rawSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTimeSeriesTbl))
|
||||
rawSrc.Where(
|
||||
rawSrc.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
rawSrc.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -883,7 +883,7 @@ func (m *module) getPerGroupDistinctCounts(
|
||||
|
||||
reducedSrc := sqlbuilder.NewSelectBuilder()
|
||||
reducedSrc.Select("labels")
|
||||
reducedSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
|
||||
reducedSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
|
||||
reducedSrc.Where(
|
||||
reducedSrc.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
reducedSrc.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -896,7 +896,7 @@ func (m *module) getPerGroupDistinctCounts(
|
||||
|
||||
sb.From(sb.BuilderAs(sqlbuilder.UnionAll(rawSrc, reducedSrc), "series"))
|
||||
} else {
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTimeSeriesTbl))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTimeSeriesTbl))
|
||||
sb.Where(
|
||||
sb.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
sb.GE("unix_milli", tsAdjustedStartMs),
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -72,7 +72,7 @@ func (m *module) getPerGroupHostStatusCounts(
|
||||
|
||||
rawSrc := sqlbuilder.NewSelectBuilder()
|
||||
rawSrc.Select("labels")
|
||||
rawSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTimeSeriesTableName))
|
||||
rawSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTimeSeriesTableName))
|
||||
rawSrc.Where(
|
||||
rawSrc.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
rawSrc.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -85,7 +85,7 @@ func (m *module) getPerGroupHostStatusCounts(
|
||||
|
||||
reducedSrc := sqlbuilder.NewSelectBuilder()
|
||||
reducedSrc.Select("labels")
|
||||
reducedSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
|
||||
reducedSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
|
||||
reducedSrc.Where(
|
||||
reducedSrc.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
reducedSrc.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -101,7 +101,7 @@ func (m *module) getPerGroupHostStatusCounts(
|
||||
|
||||
fpSB := m.buildSamplesTblFingerprintSubQuery(metricNames, localSamplesTable, samplesStartMs, flooredEndMs)
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTimeSeriesTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTimeSeriesTableName))
|
||||
sb.Where(
|
||||
sb.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
sb.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -357,7 +357,7 @@ func (m *module) getActiveHostsQuery(metricNames []string, hostNameAttr string,
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Distinct()
|
||||
sb.Select("attr_string_value")
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
|
||||
sb.Where(
|
||||
sb.In("metric_name", sqlbuilder.List(metricNames)),
|
||||
sb.E("attr_name", hostNameAttr),
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/modules/inframonitoring"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
|
||||
@@ -40,8 +40,8 @@ func NewModule(
|
||||
providerSettings factory.ProviderSettings,
|
||||
cfg inframonitoring.Config,
|
||||
) inframonitoring.Module {
|
||||
fieldMapper := metricstelemetryschema.NewFieldMapper()
|
||||
condBuilder := metricstelemetryschema.NewConditionBuilder(fieldMapper)
|
||||
fieldMapper := telemetrymetrics.NewFieldMapper()
|
||||
condBuilder := telemetrymetrics.NewConditionBuilder(fieldMapper)
|
||||
return &module{
|
||||
telemetryStore: telemetryStore,
|
||||
telemetryMetadataStore: telemetryMetadataStore,
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -186,7 +186,7 @@ func (m *module) getPerGroupNodeConditionCounts(
|
||||
|
||||
// Step-floor bounds + resolve tables in one shot to match QB v5 querier.
|
||||
samplesStartMs, flooredEndMs, tsAdjustedStartMs, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
|
||||
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
|
||||
// ----- timeSeriesFPs -----
|
||||
timeSeriesFPs := sqlbuilder.NewSelectBuilder()
|
||||
@@ -200,7 +200,7 @@ func (m *module) getPerGroupNodeConditionCounts(
|
||||
)
|
||||
}
|
||||
timeSeriesFPs.Select(timeSeriesFPsSelectCols...)
|
||||
timeSeriesFPs.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
timeSeriesFPs.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
timeSeriesFPs.Where(
|
||||
timeSeriesFPs.E("metric_name", nodeConditionMetricName),
|
||||
timeSeriesFPs.GE("unix_milli", tsAdjustedStartMs),
|
||||
@@ -237,7 +237,7 @@ func (m *module) getPerGroupNodeConditionCounts(
|
||||
latestConditionPerNode.Select(latestConditionPerNodeSelectCols...)
|
||||
latestConditionPerNode.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN time_series_fps AS tsfp ON samples.fingerprint = tsfp.fingerprint",
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
))
|
||||
latestConditionPerNode.Where(
|
||||
latestConditionPerNode.E("samples.metric_name", nodeConditionMetricName),
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -284,7 +284,7 @@ func (m *module) getPerGroupPodStatusCounts(
|
||||
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
|
||||
|
||||
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
|
||||
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
|
||||
// Build the merged filter clause once; it's identical across the three fps
|
||||
// CTEs, and buildFilterClause hits the metadata store + parses the
|
||||
@@ -313,7 +313,7 @@ func (m *module) getPerGroupPodStatusCounts(
|
||||
)
|
||||
}
|
||||
phaseFps.Select(phaseFpsCols...)
|
||||
phaseFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
phaseFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
phaseFps.Where(
|
||||
phaseFps.E("metric_name", podPhaseMetricName),
|
||||
phaseFps.GE("unix_milli", tsAdjustedStart),
|
||||
@@ -340,7 +340,7 @@ func (m *module) getPerGroupPodStatusCounts(
|
||||
phasePerPod.Select(phasePerPodCols...)
|
||||
phasePerPod.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN phase_fps AS fps ON samples.fingerprint = fps.fingerprint",
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
))
|
||||
phasePerPod.Where(
|
||||
phasePerPod.E("samples.metric_name", podPhaseMetricName),
|
||||
@@ -357,7 +357,7 @@ func (m *module) getPerGroupPodStatusCounts(
|
||||
"fingerprint",
|
||||
fmt.Sprintf("JSONExtractString(labels, %s) AS pod_uid", podReasonFps.Var(podUIDAttrKey)),
|
||||
)
|
||||
podReasonFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
podReasonFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
podReasonFps.Where(
|
||||
podReasonFps.E("metric_name", podStatusReasonMetricName),
|
||||
podReasonFps.GE("unix_milli", tsAdjustedStart),
|
||||
@@ -377,7 +377,7 @@ func (m *module) getPerGroupPodStatusCounts(
|
||||
)
|
||||
podReasonPerPod.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN pod_reason_fps AS fps ON samples.fingerprint = fps.fingerprint",
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
))
|
||||
podReasonPerPod.Where(
|
||||
podReasonPerPod.E("samples.metric_name", podStatusReasonMetricName),
|
||||
@@ -396,7 +396,7 @@ func (m *module) getPerGroupPodStatusCounts(
|
||||
fmt.Sprintf("JSONExtractString(labels, %s) AS container_name", containerReasonFps.Var(containerNameAttrKey)),
|
||||
fmt.Sprintf("JSONExtractString(labels, %s) AS reason", containerReasonFps.Var(containerStatusReasonAttrKey)),
|
||||
)
|
||||
containerReasonFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
containerReasonFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
containerReasonFps.Where(
|
||||
containerReasonFps.E("metric_name", containerStatusReasonMetricName),
|
||||
containerReasonFps.GE("unix_milli", tsAdjustedStart),
|
||||
@@ -432,7 +432,7 @@ func (m *module) getPerGroupPodStatusCounts(
|
||||
)
|
||||
containerInner.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN container_reason_fps AS fps ON samples.fingerprint = fps.fingerprint",
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
))
|
||||
containerInner.Where(
|
||||
containerInner.E("samples.metric_name", containerStatusReasonMetricName),
|
||||
@@ -611,7 +611,7 @@ func (m *module) getPerGroupPodRestartCounts(
|
||||
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
|
||||
|
||||
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
|
||||
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
|
||||
|
||||
var (
|
||||
filterClause *sqlbuilder.WhereClause
|
||||
@@ -637,7 +637,7 @@ func (m *module) getPerGroupPodRestartCounts(
|
||||
)
|
||||
}
|
||||
restartFps.Select(restartFpsCols...)
|
||||
restartFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
|
||||
restartFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
|
||||
restartFps.Where(
|
||||
restartFps.E("metric_name", containerRestartsMetricName),
|
||||
restartFps.GE("unix_milli", tsAdjustedStart),
|
||||
@@ -667,7 +667,7 @@ func (m *module) getPerGroupPodRestartCounts(
|
||||
containerRestarts.Select(containerRestartsCols...)
|
||||
containerRestarts.From(fmt.Sprintf(
|
||||
"%s.%s AS samples INNER JOIN restart_fps AS fps ON samples.fingerprint = fps.fingerprint",
|
||||
metricstelemetryschema.DBName, distributedSamplesTable,
|
||||
telemetrymetrics.DBName, distributedSamplesTable,
|
||||
))
|
||||
containerRestarts.Where(
|
||||
containerRestarts.E("samples.metric_name", containerRestartsMetricName),
|
||||
|
||||
@@ -19,8 +19,8 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/modules/dashboard"
|
||||
"github.com/SigNoz/signoz/pkg/modules/metricsexplorer"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
@@ -49,8 +49,8 @@ type module struct {
|
||||
|
||||
// NewModule constructs the metrics module with the provided dependencies.
|
||||
func NewModule(ts telemetrystore.TelemetryStore, telemetryMetadataStore telemetrytypes.MetadataStore, cache cache.Cache, ruleStore ruletypes.RuleStore, dashboardModule dashboard.Module, fl flagger.Flagger, providerSettings factory.ProviderSettings, cfg metricsexplorer.Config) metricsexplorer.Module {
|
||||
fieldMapper := metricstelemetryschema.NewFieldMapper()
|
||||
condBuilder := metricstelemetryschema.NewConditionBuilder(fieldMapper)
|
||||
fieldMapper := telemetrymetrics.NewFieldMapper()
|
||||
condBuilder := telemetrymetrics.NewConditionBuilder(fieldMapper)
|
||||
return &module{
|
||||
telemetryStore: ts,
|
||||
fieldMapper: fieldMapper,
|
||||
@@ -89,7 +89,7 @@ func (m *module) listMeterMetrics(ctx context.Context, params *metricsexplorerty
|
||||
"argMax(temporality, unix_milli) AS temporality",
|
||||
"argMax(is_monotonic, unix_milli) AS is_monotonic",
|
||||
)
|
||||
sb.From(fmt.Sprintf("%s.%s", metertelemetryschema.DBName, metertelemetryschema.SamplesTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymeter.DBName, telemetrymeter.SamplesTableName))
|
||||
|
||||
if params.Start != nil && params.End != nil {
|
||||
sb.Where(sb.Between("unix_milli", *params.Start, *params.End))
|
||||
@@ -160,11 +160,11 @@ func (m *module) listMetrics(ctx context.Context, orgID valuer.UUID, params *met
|
||||
hasRange := params.Start != nil && params.End != nil
|
||||
if hasRange {
|
||||
var distributedTsTable string
|
||||
rangeStart, rangeEnd, distributedTsTable, _ = metricstelemetryschema.WhichTSTableToUse(uint64(*params.Start), uint64(*params.End), false, nil)
|
||||
rawSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTsTable))
|
||||
rangeStart, rangeEnd, distributedTsTable, _ = telemetrymetrics.WhichTSTableToUse(uint64(*params.Start), uint64(*params.End), false, nil)
|
||||
rawSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
|
||||
rawSB.Where(rawSB.Between("unix_milli", rangeStart, rangeEnd))
|
||||
} else {
|
||||
rawSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV41weekTableName))
|
||||
rawSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV41weekTableName))
|
||||
}
|
||||
|
||||
applyListFilters(rawSB)
|
||||
@@ -174,7 +174,7 @@ func (m *module) listMetrics(ctx context.Context, orgID valuer.UUID, params *met
|
||||
if reductionEnabled {
|
||||
reducedSB := sqlbuilder.NewSelectBuilder()
|
||||
reducedSB.Select("DISTINCT metric_name")
|
||||
reducedSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
|
||||
reducedSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
|
||||
if hasRange {
|
||||
reducedSB.Where(reducedSB.Between("unix_milli", rangeStart, rangeEnd))
|
||||
}
|
||||
@@ -512,7 +512,7 @@ func (m *module) CheckMetricExists(ctx context.Context, orgID valuer.UUID, metri
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("count(*) > 0 as metricExists")
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
|
||||
sb.Where(sb.E("metric_name", metricName))
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
@@ -534,7 +534,7 @@ func (m *module) HasNonSigNozMetrics(ctx context.Context) (bool, error) {
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("count(*) > 0")
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV41weekTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV41weekTableName))
|
||||
sb.Where("metric_name NOT LIKE 'signoz_%'")
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
@@ -570,10 +570,10 @@ func (m *module) InspectMetrics(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tsStart, _, tsTable, _ := metricstelemetryschema.WhichTSTableToUse(start, end, false, nil)
|
||||
tsStart, _, tsTable, _ := telemetrymetrics.WhichTSTableToUse(start, end, false, nil)
|
||||
tsSb := sqlbuilder.NewSelectBuilder()
|
||||
tsSb.Select("fingerprint", "labels")
|
||||
tsSb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, tsTable))
|
||||
tsSb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, tsTable))
|
||||
tsSb.Where(
|
||||
tsSb.E("metric_name", req.MetricName),
|
||||
tsSb.GE("unix_milli", tsStart),
|
||||
@@ -625,7 +625,7 @@ func (m *module) InspectMetrics(
|
||||
|
||||
samplesSb := sqlbuilder.NewSelectBuilder()
|
||||
samplesSb.Select("fingerprint", "unix_milli", "value")
|
||||
samplesSb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4TableName))
|
||||
samplesSb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4TableName))
|
||||
samplesSb.Where(
|
||||
samplesSb.In("fingerprint", fingerprints...),
|
||||
samplesSb.E("metric_name", req.MetricName),
|
||||
@@ -717,7 +717,7 @@ func (m *module) fetchUpdatedMetadata(ctx context.Context, orgID valuer.UUID, me
|
||||
"argMax(temporality, created_at) AS temporality",
|
||||
"argMax(is_monotonic, created_at) AS is_monotonic",
|
||||
)
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.UpdatedMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.UpdatedMetadataTableName))
|
||||
sb.Where(sb.In("metric_name", args...))
|
||||
sb.GroupBy("metric_name")
|
||||
|
||||
@@ -777,7 +777,7 @@ func (m *module) fetchTimeseriesMetadata(ctx context.Context, orgID valuer.UUID,
|
||||
"anyLast(temporality) AS temporality",
|
||||
"argMax(is_monotonic, unix_milli) AS is_monotonic",
|
||||
)
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4TableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4TableName))
|
||||
sb.Where(sb.In("metric_name", args...))
|
||||
sb.GroupBy("metric_name")
|
||||
|
||||
@@ -891,7 +891,7 @@ func (m *module) checkForLabelInMetric(ctx context.Context, metricName string, l
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("count(*) > 0 AS has_label")
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
|
||||
sb.Where(sb.E("metric_name", metricName))
|
||||
sb.Where(sb.E("attr_name", label))
|
||||
sb.Limit(1)
|
||||
@@ -914,7 +914,7 @@ func (m *module) insertMetricsMetadata(ctx context.Context, orgID valuer.UUID, r
|
||||
createdAt := time.Now().UnixMilli()
|
||||
|
||||
ib := sqlbuilder.NewInsertBuilder()
|
||||
ib.InsertInto(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.UpdatedMetadataTableName))
|
||||
ib.InsertInto(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.UpdatedMetadataTableName))
|
||||
ib.Cols("metric_name", "temporality", "is_monotonic", "type", "description", "unit", "created_at")
|
||||
ib.Values(
|
||||
req.MetricName,
|
||||
@@ -1016,9 +1016,9 @@ func (m *module) fetchMetricsStatsWithSamples(
|
||||
}
|
||||
}
|
||||
|
||||
start, end, distributedTsTable, localTsTable := metricstelemetryschema.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
|
||||
distributedSamplesTable, _ := metricstelemetryschema.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil)
|
||||
countExp := metricstelemetryschema.CountExpressionForSamplesTable(distributedSamplesTable)
|
||||
start, end, distributedTsTable, localTsTable := telemetrymetrics.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
|
||||
distributedSamplesTable, _ := telemetrymetrics.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil)
|
||||
countExp := telemetrymetrics.CountExpressionForSamplesTable(distributedSamplesTable)
|
||||
|
||||
// Timeseries counts per metric
|
||||
tsSB := sqlbuilder.NewSelectBuilder()
|
||||
@@ -1026,7 +1026,7 @@ func (m *module) fetchMetricsStatsWithSamples(
|
||||
"metric_name",
|
||||
"uniq(fingerprint) AS timeseries",
|
||||
)
|
||||
tsSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTsTable))
|
||||
tsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
|
||||
tsSB.Where(tsSB.Between("unix_milli", start, end))
|
||||
tsSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
if filterWhereClause != nil {
|
||||
@@ -1040,7 +1040,7 @@ func (m *module) fetchMetricsStatsWithSamples(
|
||||
"metric_name",
|
||||
fmt.Sprintf("%s AS samples", countExp),
|
||||
)
|
||||
samplesSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedSamplesTable))
|
||||
samplesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedSamplesTable))
|
||||
samplesSB.Where(samplesSB.Between("unix_milli", req.Start, req.End))
|
||||
samplesSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
|
||||
@@ -1053,7 +1053,7 @@ func (m *module) fetchMetricsStatsWithSamples(
|
||||
if filterWhereClause != nil {
|
||||
fingerprintSB := sqlbuilder.NewSelectBuilder()
|
||||
fingerprintSB.Select("fingerprint")
|
||||
fingerprintSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTsTable))
|
||||
fingerprintSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTsTable))
|
||||
fingerprintSB.Where(fingerprintSB.Between("unix_milli", start, end))
|
||||
fingerprintSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
fingerprintSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause))
|
||||
@@ -1064,7 +1064,7 @@ func (m *module) fetchMetricsStatsWithSamples(
|
||||
} else {
|
||||
metricNamesSB := sqlbuilder.NewSelectBuilder()
|
||||
metricNamesSB.Select("DISTINCT metric_name")
|
||||
metricNamesSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTsTable))
|
||||
metricNamesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTsTable))
|
||||
metricNamesSB.Where(metricNamesSB.Between("unix_milli", start, end))
|
||||
metricNamesSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
|
||||
@@ -1090,18 +1090,18 @@ func (m *module) fetchMetricsStatsWithSamples(
|
||||
// the data lands either of the _reduced_last/_reduced_sum tables
|
||||
reducedLastSB := sqlbuilder.NewSelectBuilder()
|
||||
reducedLastSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt")
|
||||
reducedLastSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedLastTableName))
|
||||
reducedLastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName))
|
||||
reducedLastSB.Where(reducedLastSB.Between("unix_milli", req.Start, req.End))
|
||||
reducedLastSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
|
||||
reducedSumSB := sqlbuilder.NewSelectBuilder()
|
||||
reducedSumSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt")
|
||||
reducedSumSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedSumTableName))
|
||||
reducedSumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName))
|
||||
reducedSumSB.Where(reducedSumSB.Between("unix_milli", req.Start, req.End))
|
||||
reducedSumSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
|
||||
// separate query for reduced series counts
|
||||
reducedTsSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
|
||||
reducedTsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
|
||||
reducedTsSB.Where(reducedTsSB.Between("unix_milli", start, end))
|
||||
reducedTsSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
reducedTsSB.GroupBy("metric_name")
|
||||
@@ -1112,7 +1112,7 @@ func (m *module) fetchMetricsStatsWithSamples(
|
||||
// samples uses a separate cte with local table
|
||||
reducedFpSB := sqlbuilder.NewSelectBuilder()
|
||||
reducedFpSB.Select("fingerprint")
|
||||
reducedFpSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedLocalTableName))
|
||||
reducedFpSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedLocalTableName))
|
||||
reducedFpSB.Where(reducedFpSB.Between("unix_milli", start, end))
|
||||
reducedFpSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
reducedFpSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause))
|
||||
@@ -1220,11 +1220,11 @@ func (m *module) computeTimeseriesTreemap(ctx context.Context, orgID valuer.UUID
|
||||
}
|
||||
}
|
||||
|
||||
start, end, distributedTsTable, _ := metricstelemetryschema.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
|
||||
start, end, distributedTsTable, _ := telemetrymetrics.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
|
||||
|
||||
totalTSBuilder := sqlbuilder.NewSelectBuilder()
|
||||
totalTSBuilder.Select("uniq(fingerprint) AS total_time_series")
|
||||
totalTSBuilder.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTsTable))
|
||||
totalTSBuilder.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
|
||||
totalTSBuilder.Where(totalTSBuilder.Between("unix_milli", start, end))
|
||||
|
||||
metricsSB := sqlbuilder.NewSelectBuilder()
|
||||
@@ -1232,7 +1232,7 @@ func (m *module) computeTimeseriesTreemap(ctx context.Context, orgID valuer.UUID
|
||||
"metric_name",
|
||||
"uniq(fingerprint) AS total_value",
|
||||
)
|
||||
metricsSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTsTable))
|
||||
metricsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
|
||||
metricsSB.Where(metricsSB.Between("unix_milli", start, end))
|
||||
metricsSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
if filterWhereClause != nil {
|
||||
@@ -1244,7 +1244,7 @@ func (m *module) computeTimeseriesTreemap(ctx context.Context, orgID valuer.UUID
|
||||
if reductionEnabled {
|
||||
reducedTotalTSBuilder := sqlbuilder.NewSelectBuilder()
|
||||
reducedTotalTSBuilder.Select("uniq(fingerprint) AS total_time_series")
|
||||
reducedTotalTSBuilder.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
|
||||
reducedTotalTSBuilder.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
|
||||
reducedTotalTSBuilder.Where(reducedTotalTSBuilder.Between("unix_milli", start, end))
|
||||
|
||||
reducedMetricsSB := sqlbuilder.NewSelectBuilder()
|
||||
@@ -1252,7 +1252,7 @@ func (m *module) computeTimeseriesTreemap(ctx context.Context, orgID valuer.UUID
|
||||
"metric_name",
|
||||
"uniq(fingerprint) AS total_value",
|
||||
)
|
||||
reducedMetricsSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
|
||||
reducedMetricsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
|
||||
reducedMetricsSB.Where(reducedMetricsSB.Between("unix_milli", start, end))
|
||||
reducedMetricsSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
if filterWhereClause != nil {
|
||||
@@ -1338,15 +1338,15 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
|
||||
}
|
||||
}
|
||||
|
||||
start, end, distributedTsTable, localTsTable := metricstelemetryschema.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
|
||||
distributedSamplesTable, _ := metricstelemetryschema.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil)
|
||||
countExp := metricstelemetryschema.CountExpressionForSamplesTable(distributedSamplesTable)
|
||||
start, end, distributedTsTable, localTsTable := telemetrymetrics.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
|
||||
distributedSamplesTable, _ := telemetrymetrics.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil)
|
||||
countExp := telemetrymetrics.CountExpressionForSamplesTable(distributedSamplesTable)
|
||||
|
||||
candidateLimit := req.Limit + 50
|
||||
|
||||
metricCandidatesSB := sqlbuilder.NewSelectBuilder()
|
||||
metricCandidatesSB.Select("metric_name")
|
||||
metricCandidatesSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTsTable))
|
||||
metricCandidatesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
|
||||
metricCandidatesSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
metricCandidatesSB.Where(metricCandidatesSB.Between("unix_milli", start, end))
|
||||
if filterWhereClause != nil {
|
||||
@@ -1364,7 +1364,7 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
|
||||
if reductionEnabled {
|
||||
reducedCandidatesSB = sqlbuilder.NewSelectBuilder()
|
||||
reducedCandidatesSB.Select("metric_name")
|
||||
reducedCandidatesSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
|
||||
reducedCandidatesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
|
||||
reducedCandidatesSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
reducedCandidatesSB.Where(reducedCandidatesSB.Between("unix_milli", start, end))
|
||||
if filterWhereClause != nil {
|
||||
@@ -1379,7 +1379,7 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
|
||||
|
||||
totalSamplesSB := sqlbuilder.NewSelectBuilder()
|
||||
totalSamplesSB.Select(fmt.Sprintf("%s AS total_samples", countExp))
|
||||
totalSamplesSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedSamplesTable))
|
||||
totalSamplesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedSamplesTable))
|
||||
totalSamplesSB.Where(totalSamplesSB.Between("unix_milli", req.Start, req.End))
|
||||
|
||||
sampleCountsSB := sqlbuilder.NewSelectBuilder()
|
||||
@@ -1387,7 +1387,7 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
|
||||
"metric_name",
|
||||
fmt.Sprintf("%s AS samples", countExp),
|
||||
)
|
||||
sampleCountsSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedSamplesTable))
|
||||
sampleCountsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedSamplesTable))
|
||||
sampleCountsSB.Where(sampleCountsSB.Between("unix_milli", req.Start, req.End))
|
||||
sampleCountsSB.Where("metric_name GLOBAL IN (SELECT metric_name FROM __metric_candidates)")
|
||||
|
||||
@@ -1397,13 +1397,13 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
|
||||
if reductionEnabled {
|
||||
reducedLastSB = sqlbuilder.NewSelectBuilder()
|
||||
reducedLastSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt")
|
||||
reducedLastSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedLastTableName))
|
||||
reducedLastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName))
|
||||
reducedLastSB.Where(reducedLastSB.Between("unix_milli", req.Start, req.End))
|
||||
reducedLastSB.Where("metric_name GLOBAL IN (SELECT metric_name FROM __reduced_metric_candidates)")
|
||||
|
||||
reducedSumSB = sqlbuilder.NewSelectBuilder()
|
||||
reducedSumSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt")
|
||||
reducedSumSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedSumTableName))
|
||||
reducedSumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName))
|
||||
reducedSumSB.Where(reducedSumSB.Between("unix_milli", req.Start, req.End))
|
||||
reducedSumSB.Where("metric_name GLOBAL IN (SELECT metric_name FROM __reduced_metric_candidates)")
|
||||
}
|
||||
@@ -1411,7 +1411,7 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
|
||||
if filterWhereClause != nil {
|
||||
fingerprintSB := sqlbuilder.NewSelectBuilder()
|
||||
fingerprintSB.Select("fingerprint")
|
||||
fingerprintSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTsTable))
|
||||
fingerprintSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTsTable))
|
||||
fingerprintSB.Where(fingerprintSB.Between("unix_milli", start, end))
|
||||
fingerprintSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
fingerprintSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause))
|
||||
@@ -1423,7 +1423,7 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
|
||||
if reductionEnabled {
|
||||
reducedFingerprintSB := sqlbuilder.NewSelectBuilder()
|
||||
reducedFingerprintSB.Select("fingerprint")
|
||||
reducedFingerprintSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedLocalTableName))
|
||||
reducedFingerprintSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedLocalTableName))
|
||||
reducedFingerprintSB.Where(reducedFingerprintSB.Between("unix_milli", start, end))
|
||||
reducedFingerprintSB.Where("NOT startsWith(metric_name, 'signoz')")
|
||||
reducedFingerprintSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause))
|
||||
@@ -1457,12 +1457,12 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
|
||||
// Grand total includes reduced sample volume so percentages stay consistent.
|
||||
reducedTotalLastSB := sqlbuilder.NewSelectBuilder()
|
||||
reducedTotalLastSB.Select("uniq(reduced_fingerprint, unix_milli) AS cnt")
|
||||
reducedTotalLastSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedLastTableName))
|
||||
reducedTotalLastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName))
|
||||
reducedTotalLastSB.Where(reducedTotalLastSB.Between("unix_milli", req.Start, req.End))
|
||||
|
||||
reducedTotalSumSB := sqlbuilder.NewSelectBuilder()
|
||||
reducedTotalSumSB.Select("uniq(reduced_fingerprint, unix_milli) AS cnt")
|
||||
reducedTotalSumSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedSumTableName))
|
||||
reducedTotalSumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName))
|
||||
reducedTotalSumSB.Where(reducedTotalSumSB.Between("unix_milli", req.Start, req.End))
|
||||
|
||||
reducedTotalSamplesSB := sqlbuilder.NewSelectBuilder()
|
||||
@@ -1552,7 +1552,7 @@ func (m *module) getMetricDataPoints(ctx context.Context, metricName string) (ui
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("sum(count) AS data_points")
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4Agg30mTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4Agg30mTableName))
|
||||
sb.Where(sb.E("metric_name", metricName))
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
@@ -1574,7 +1574,7 @@ func (m *module) getMetricLastReceived(ctx context.Context, metricName string) (
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("MAX(last_reported_unix_milli) AS last_received_time")
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
|
||||
sb.Where(sb.E("metric_name", metricName))
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
@@ -1599,7 +1599,7 @@ func (m *module) getTotalTimeSeriesForMetricName(ctx context.Context, metricName
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("uniq(fingerprint) AS time_series_count")
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV41weekTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV41weekTableName))
|
||||
sb.Where(sb.E("metric_name", metricName))
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
@@ -1623,7 +1623,7 @@ func (m *module) getActiveTimeSeriesForMetricName(ctx context.Context, metricNam
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("uniq(fingerprint) AS active_time_series")
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4TableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4TableName))
|
||||
sb.Where(sb.E("metric_name", metricName))
|
||||
sb.Where(sb.GTE("unix_milli", milli))
|
||||
|
||||
@@ -1649,7 +1649,7 @@ func (m *module) fetchMetricAttributes(ctx context.Context, metricName string, s
|
||||
"groupUniqArray(1000)(attr_string_value) AS values",
|
||||
"uniq(attr_string_value) AS valueCount",
|
||||
)
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
|
||||
sb.Where(sb.E("metric_name", metricName))
|
||||
sb.Where("NOT startsWith(attr_name, '__')")
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/modules/promote"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
@@ -52,7 +52,7 @@ func (m *module) ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetype
|
||||
|
||||
response := []promotetypes.PromotePath{}
|
||||
for _, path := range promotedPaths {
|
||||
fullPath := logstelemetryschema.BodyPromotedColumnPrefix + path
|
||||
fullPath := telemetrylogs.BodyPromotedColumnPrefix + path
|
||||
path = telemetrytypes.BodyJSONStringSearchPrefix + path
|
||||
item := promotetypes.PromotePath{
|
||||
Path: path,
|
||||
@@ -68,7 +68,7 @@ func (m *module) ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetype
|
||||
|
||||
// add the paths that are not promoted but have indexes
|
||||
for path, indexes := range aggr {
|
||||
path := strings.TrimPrefix(path, logstelemetryschema.BodyV2ColumnPrefix)
|
||||
path := strings.TrimPrefix(path, telemetrylogs.BodyV2ColumnPrefix)
|
||||
path = telemetrytypes.BodyJSONStringSearchPrefix + path
|
||||
response = append(response, promotetypes.PromotePath{
|
||||
Path: path,
|
||||
@@ -108,8 +108,8 @@ func (m *module) createIndexes(ctx context.Context, indexes []schemamigrator.Ind
|
||||
|
||||
for _, index := range indexes {
|
||||
alterStmt := schemamigrator.AlterTableAddIndex{
|
||||
Database: logstelemetryschema.DBName,
|
||||
Table: logstelemetryschema.LogsV2LocalTableName,
|
||||
Database: telemetrylogs.DBName,
|
||||
Table: telemetrylogs.LogsV2LocalTableName,
|
||||
Index: index,
|
||||
}
|
||||
op := alterStmt.OnCluster(m.telemetryStore.Cluster())
|
||||
@@ -153,10 +153,10 @@ func (m *module) PromoteAndIndexPaths(
|
||||
}
|
||||
}
|
||||
if len(it.Indexes) > 0 {
|
||||
parentColumn := logstelemetryschema.LogsV2BodyV2Column
|
||||
parentColumn := telemetrylogs.LogsV2BodyV2Column
|
||||
// if the path is already promoted or is being promoted, add it to the promoted column
|
||||
if _, promoted := existingPromotedPaths[it.Path]; promoted || it.Promote {
|
||||
parentColumn = logstelemetryschema.LogsV2BodyPromotedColumn
|
||||
parentColumn = telemetrylogs.LogsV2BodyPromotedColumn
|
||||
}
|
||||
|
||||
for _, index := range it.Indexes {
|
||||
|
||||
@@ -11,8 +11,8 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/modules/services"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -39,7 +39,7 @@ func (m *module) FetchTopLevelOperations(ctx context.Context, start time.Time, s
|
||||
ctx = m.withServicesContext(ctx, "FetchTopLevelOperations")
|
||||
|
||||
db := m.TelemetryStore.ClickhouseDB()
|
||||
query := fmt.Sprintf("SELECT name, serviceName, max(time) as ts FROM %s.%s WHERE time >= @start", tracestelemetryschema.DBName, tracestelemetryschema.TopLevelOperationsTableName)
|
||||
query := fmt.Sprintf("SELECT name, serviceName, max(time) as ts FROM %s.%s WHERE time >= @start", telemetrytraces.DBName, telemetrytraces.TopLevelOperationsTableName)
|
||||
args := []any{clickhouse.Named("start", start)}
|
||||
if len(services) > 0 {
|
||||
query += " AND serviceName IN @services"
|
||||
|
||||
89
pkg/prometheus/clickhouseprometheusv2/capture.go
Normal file
89
pkg/prometheus/clickhouseprometheusv2/capture.go
Normal file
@@ -0,0 +1,89 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/util/annotations"
|
||||
)
|
||||
|
||||
// statementRecorder collects the statements a PromQL evaluation would run.
|
||||
// Safe for concurrent use: the engine may Select selectors concurrently.
|
||||
type statementRecorder struct {
|
||||
mu sync.Mutex
|
||||
statements []prometheus.CapturedStatement
|
||||
}
|
||||
|
||||
func (r *statementRecorder) record(query string, args []any) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.statements = append(r.statements, prometheus.CapturedStatement{Query: query, Args: args})
|
||||
}
|
||||
|
||||
func (r *statementRecorder) Statements() []prometheus.CapturedStatement {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
out := make([]prometheus.CapturedStatement, len(r.statements))
|
||||
copy(out, r.statements)
|
||||
return out
|
||||
}
|
||||
|
||||
type captureQueryable struct {
|
||||
client *client
|
||||
recorder *statementRecorder
|
||||
}
|
||||
|
||||
func (c *captureQueryable) Querier(mint, maxt int64) (storage.Querier, error) {
|
||||
return &captureQuerier{
|
||||
querier: querier{mint: mint, maxt: maxt, client: c.client},
|
||||
recorder: c.recorder,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// captureQuerier builds the same SQL as the live querier but records it and
|
||||
// returns no data. The fingerprint filter always takes the subquery form:
|
||||
// without executing the series lookup, the inline literal set is unknown.
|
||||
type captureQuerier struct {
|
||||
querier
|
||||
recorder *statementRecorder
|
||||
}
|
||||
|
||||
func (c *captureQuerier) Select(ctx context.Context, _ bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet {
|
||||
if rawQuery, ok := rawSQLQuery(matchers); ok {
|
||||
c.recorder.record(rawQuery, nil)
|
||||
return storage.EmptySeriesSet()
|
||||
}
|
||||
|
||||
start, end := c.window(hints)
|
||||
|
||||
samplesQuery, args, err := buildSamplesQuery(start, end, metricNamesFromMatchers(matchers), nil, matchers, c.lastSamplePerStepFor(ctx, hints))
|
||||
if err != nil {
|
||||
return storage.ErrSeriesSet(err)
|
||||
}
|
||||
c.recorder.record(samplesQuery, args)
|
||||
|
||||
return storage.EmptySeriesSet()
|
||||
}
|
||||
|
||||
func (c *captureQuerier) LabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
func (c *captureQuerier) LabelNames(context.Context, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
// metricNamesFromMatchers extracts the statically known metric name, if any.
|
||||
// The live path derives names from the matched series; the capture path has
|
||||
// no execution results, so only a __name__ equality contributes.
|
||||
func metricNamesFromMatchers(matchers []*labels.Matcher) []string {
|
||||
for _, m := range matchers {
|
||||
if m.Name == metricNameLabel && m.Type == labels.MatchEqual && m.Value != "" {
|
||||
return []string{m.Value}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
282
pkg/prometheus/clickhouseprometheusv2/client.go
Normal file
282
pkg/prometheus/clickhouseprometheusv2/client.go
Normal file
@@ -0,0 +1,282 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"slices"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
promValue "github.com/prometheus/prometheus/model/value"
|
||||
)
|
||||
|
||||
// seriesLookup is a series-lookup result: matched fingerprints with their
|
||||
// labels, and the distinct metric names seen on them.
|
||||
type seriesLookup struct {
|
||||
fingerprints map[uint64]labels.Labels
|
||||
metricNames []string
|
||||
}
|
||||
|
||||
// client executes the series, samples and raw queries against ClickHouse.
|
||||
type client struct {
|
||||
settings factory.ScopedProviderSettings
|
||||
telemetryStore telemetrystore.TelemetryStore
|
||||
cfg prometheus.ClickhouseV2Config
|
||||
lookbackMs int64
|
||||
}
|
||||
|
||||
func newClient(settings factory.ScopedProviderSettings, telemetryStore telemetrystore.TelemetryStore, cfg prometheus.Config) *client {
|
||||
lookback := cfg.LookbackDelta
|
||||
if lookback <= 0 {
|
||||
// Mirror the engine: promql defaults an unset lookback to 5m.
|
||||
lookback = defaultLookbackDelta
|
||||
}
|
||||
return &client{
|
||||
settings: settings,
|
||||
telemetryStore: telemetryStore,
|
||||
cfg: cfg.ClickhouseV2,
|
||||
lookbackMs: lookback.Milliseconds(),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *client) withContext(ctx context.Context, functionName string) context.Context {
|
||||
return ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
|
||||
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalMetrics.StringValue(),
|
||||
instrumentationtypes.CodeNamespace: "clickhouse-prometheus-v2",
|
||||
instrumentationtypes.CodeFunctionName: functionName,
|
||||
})
|
||||
}
|
||||
|
||||
// selectSeries runs the series lookup for the given matchers and window.
|
||||
func (c *client) selectSeries(ctx context.Context, query string, args []any) (*seriesLookup, error) {
|
||||
ctx = c.withContext(ctx, "selectSeries")
|
||||
rows, err := c.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
lookup := &seriesLookup{fingerprints: make(map[uint64]labels.Labels)}
|
||||
names := make(map[string]struct{})
|
||||
|
||||
var fingerprint uint64
|
||||
var labelsJSON string
|
||||
for rows.Next() {
|
||||
if err := rows.Scan(&fingerprint, &labelsJSON); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lset, err := unmarshalLabels(labelsJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lookup.fingerprints[fingerprint] = lset
|
||||
if name := lset.Get(metricNameLabel); name != "" {
|
||||
names[name] = struct{}{}
|
||||
}
|
||||
if c.cfg.MaxFetchedSeries > 0 && len(lookup.fingerprints) > c.cfg.MaxFetchedSeries {
|
||||
return nil, errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
"promql selector matched more than %d series; narrow the label matchers or raise prometheus::clickhousev2::max_fetched_series",
|
||||
c.cfg.MaxFetchedSeries,
|
||||
)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for name := range names {
|
||||
lookup.metricNames = append(lookup.metricNames, name)
|
||||
}
|
||||
slices.Sort(lookup.metricNames)
|
||||
|
||||
return lookup, nil
|
||||
}
|
||||
|
||||
// unmarshalLabels parses the labels JSON column. Unlike v1, the fingerprint
|
||||
// is not injected as a synthetic label (it would take part in `without (...)`
|
||||
// grouping and vector matching) and empty-valued labels are dropped: an empty
|
||||
// label value means "label absent" in Prometheus, and upstream never produces
|
||||
// such labels, but stored attribute JSON can carry them.
|
||||
func unmarshalLabels(s string) (labels.Labels, error) {
|
||||
m := make(map[string]string)
|
||||
if err := json.Unmarshal([]byte(s), &m); err != nil {
|
||||
return labels.EmptyLabels(), err
|
||||
}
|
||||
builder := labels.NewScratchBuilder(len(m))
|
||||
for k, v := range m {
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
builder.Add(k, v)
|
||||
}
|
||||
builder.Sort()
|
||||
return builder.Labels(), nil
|
||||
}
|
||||
|
||||
// selectSamples executes a samples query (raw or last-sample-per-step; both
|
||||
// produce the same column shape) and assembles the per-series sample slices.
|
||||
// Rows arrive ordered by (fingerprint, unix_milli). Rows whose fingerprint
|
||||
// is missing from the lookup are skipped (possible in the subquery filter
|
||||
// mode, where the fingerprint filter re-runs after the lookup and can see
|
||||
// series born in between). Stale flags map to the engine's StaleNaN.
|
||||
// Duplicate timestamps pass through as stored: upstream Prometheus cannot
|
||||
// produce them (its TSDB rejects them at ingest), our ingest can under
|
||||
// at-least-once retries, and v1 feeds them to the engine as-is —
|
||||
// deduplicating here would make this provider silently disagree with both
|
||||
// v1 and the transpiled statements over the same dirty data. Uniqueness
|
||||
// belongs to the ingest layer.
|
||||
func (c *client) selectSamples(ctx context.Context, query string, args []any, lookup *seriesLookup) ([]*series, error) {
|
||||
ctx = c.withContext(ctx, "selectSamples")
|
||||
rows, err := c.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var (
|
||||
result []*series
|
||||
current *series
|
||||
fingerprint uint64
|
||||
prevFp uint64
|
||||
timestampMs int64
|
||||
val float64
|
||||
flags uint32
|
||||
first = true
|
||||
haveCurrent bool
|
||||
staleMarker = math.Float64frombits(promValue.StaleNaN)
|
||||
maxSamples = c.cfg.MaxFetchedSamples
|
||||
fetched int64
|
||||
unknownCount int
|
||||
)
|
||||
|
||||
for rows.Next() {
|
||||
if err := rows.Scan(&fingerprint, ×tampMs, &val, &flags); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fetched++
|
||||
if maxSamples > 0 && fetched > maxSamples {
|
||||
return nil, errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
"promql query would fetch more than %d samples; narrow the selector or time range, or raise prometheus::clickhousev2::max_fetched_samples",
|
||||
maxSamples,
|
||||
)
|
||||
}
|
||||
|
||||
if first || fingerprint != prevFp {
|
||||
first = false
|
||||
prevFp = fingerprint
|
||||
lset, ok := lookup.fingerprints[fingerprint]
|
||||
if !ok {
|
||||
unknownCount++
|
||||
haveCurrent = false
|
||||
continue
|
||||
}
|
||||
current = &series{lset: lset}
|
||||
result = append(result, current)
|
||||
haveCurrent = true
|
||||
}
|
||||
if !haveCurrent {
|
||||
// Remaining rows of a fingerprint missing from the lookup.
|
||||
continue
|
||||
}
|
||||
|
||||
if flags&1 == 1 {
|
||||
val = staleMarker
|
||||
}
|
||||
current.ts = append(current.ts, timestampMs)
|
||||
current.vs = append(current.vs, val)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if unknownCount > 0 {
|
||||
c.settings.Logger().DebugContext(ctx, "skipped samples of fingerprints missing from series lookup",
|
||||
slog.Int("unknown_fingerprints", unknownCount))
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// queryRaw supports the {job="rawsql", query="..."} escape hatch: the value of
|
||||
// the query matcher runs as-is, each row becoming a single-sample series
|
||||
// stamped at the query end. Column "value" is the sample value; every other
|
||||
// column is a label.
|
||||
func (c *client) queryRaw(ctx context.Context, query string, ts int64) ([]*series, error) {
|
||||
ctx = c.withContext(ctx, "queryRaw")
|
||||
rows, err := c.telemetryStore.ClickhouseDB().Query(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
columns := rows.Columns()
|
||||
targets := make([]any, len(columns))
|
||||
for i := range targets {
|
||||
targets[i] = new(scanner)
|
||||
}
|
||||
|
||||
var result []*series
|
||||
for rows.Next() {
|
||||
if err := rows.Scan(targets...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
builder := labels.NewScratchBuilder(len(columns))
|
||||
var val float64
|
||||
for i, col := range columns {
|
||||
v := targets[i].(*scanner)
|
||||
if col == "value" {
|
||||
val = v.f
|
||||
continue
|
||||
}
|
||||
builder.Add(col, v.s)
|
||||
}
|
||||
builder.Sort()
|
||||
result = append(result, &series{
|
||||
lset: builder.Labels(),
|
||||
ts: []int64{ts},
|
||||
vs: []float64{val},
|
||||
})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
var _ sql.Scanner = (*scanner)(nil)
|
||||
|
||||
type scanner struct {
|
||||
f float64
|
||||
s string
|
||||
}
|
||||
|
||||
func (s *scanner) Scan(val any) error {
|
||||
s.f = 0
|
||||
s.s = ""
|
||||
|
||||
s.s = fmt.Sprintf("%v", val)
|
||||
switch val := val.(type) {
|
||||
case int64:
|
||||
s.f = float64(val)
|
||||
case uint64:
|
||||
s.f = float64(val)
|
||||
case float64:
|
||||
s.f = val
|
||||
case []byte:
|
||||
s.s = string(val)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
86
pkg/prometheus/clickhouseprometheusv2/doc.go
Normal file
86
pkg/prometheus/clickhouseprometheusv2/doc.go
Normal file
@@ -0,0 +1,86 @@
|
||||
// Package clickhouseprometheusv2 is the second-generation ClickHouse-backed
|
||||
// Prometheus provider. It exists because the v1 provider fetches every raw
|
||||
// sample of a query's union window through the remote-read protobuf layer
|
||||
// and hands it to the engine — the cost is a function of ingested data, not
|
||||
// of the question asked, which is how a dashboard of PromQL panels takes an
|
||||
// instance down.
|
||||
//
|
||||
// Here the stock promql engine evaluates over this package's native
|
||||
// storage.Querier: no remote-read translation layer, per-selector fetch
|
||||
// windows instead of the query-wide union, and fetch reductions that are
|
||||
// provably invisible to the engine. Correctness is the constraint that
|
||||
// shaped everything: a PromQL result that differs from upstream Prometheus
|
||||
// is a lost user, so every reduction below either preserves engine semantics
|
||||
// exactly or is not performed.
|
||||
//
|
||||
// # Series lookup
|
||||
//
|
||||
// Matchers resolve to series once per selector (selectSeries): __name__
|
||||
// matchers translate to the metric_name column — all four matcher types; the
|
||||
// v1 client silently returned nothing for regex metric names — and every
|
||||
// other matcher to a JSONExtractString condition on the labels column
|
||||
// (applySeriesConditions). Regexes are anchored before they reach match():
|
||||
// PromQL matchers match the whole value, ClickHouse match() searches for a
|
||||
// substring, and without anchoring =~"api" would also select "x-api-y". An
|
||||
// equality matcher against "" matches series without the label, mirroring
|
||||
// PromQL, because JSONExtractString returns "" for missing keys. The series
|
||||
// tables hold one row per (fingerprint, bucket) at 1h/6h/1d/1w
|
||||
// granularities; timeSeriesTableFor picks the table whose bucket fits the
|
||||
// window and rounds the window start down to the bucket boundary. The
|
||||
// resulting label sets drop what v1 leaked into results: the synthetic
|
||||
// fingerprint label (it would take part in without() grouping and vector
|
||||
// matching) and empty-valued labels. MaxFetchedSeries fails the lookup with
|
||||
// a typed invalid-input error past the ceiling — v1's behavior for an
|
||||
// oversized selector was to buffer everything and OOM, and a 4xx the user
|
||||
// can narrow beats a dead process serving nobody.
|
||||
//
|
||||
// # Sample fetch
|
||||
//
|
||||
// Samples are fetched per selector using the engine's per-selector hints,
|
||||
// not the query-wide union window, so foo / foo offset 1d reads two narrow
|
||||
// windows instead of the widest one twice. Instant selectors of
|
||||
// subquery-free queries fetch only the last sample per step bucket
|
||||
// (lastSamplePerStep): the engine resolves an instant selector at each grid
|
||||
// timestamp t to the latest sample in the left-open lookback window
|
||||
// (t - lookback, t], so buckets anchor at the selector's first evaluation
|
||||
// timestamp — recovered from the hints as hints.Start + lookback - 1ms, the
|
||||
// inverse of how the engine derives hints.Start — and bucket boundaries
|
||||
// coincide with evaluation timestamps: a non-final sample of a bucket can
|
||||
// never be the latest sample in (t - lookback, t] for any grid t. Real
|
||||
// timestamps are preserved, so the engine's own lookback and staleness
|
||||
// handling stay exact. Range selectors always fetch raw — every sample feeds
|
||||
// the range function — and the subquery-free proof travels in the context as
|
||||
// prometheus.QueryTraits, because subquery selectors evaluate at the
|
||||
// subquery's step while the hints carry the top-level step; call sites that
|
||||
// do not attach traits get the conservative raw fetch. Row assembly counts
|
||||
// rows against MaxFetchedSamples while scanning, keeps the first of
|
||||
// consecutive equal timestamps, maps stale flags to the engine's StaleNaN,
|
||||
// and merges series with identical label sets (sortAndMerge) — the engine
|
||||
// assumes storages never emit duplicates. A {job="rawsql", query="..."}
|
||||
// selector bypasses all of this and runs the query matcher's value verbatim.
|
||||
//
|
||||
// # Sharding
|
||||
//
|
||||
// samples_v4 and time_series_v4 (and all their rollups) shard on the same
|
||||
// key — cityHash64(env, temporality, metric_name, fingerprint) — so a
|
||||
// series' samples and catalog rows live on the same shard. Fingerprint
|
||||
// filters exploit that: matched sets inline as sorted literals up to
|
||||
// inlineFingerprintsLimit (literals engage the samples primary key; sorting
|
||||
// keeps statements deterministic), beyond it the filter is a shard-local
|
||||
// series subquery on the same predicates, not a GLOBAL broadcast of the
|
||||
// matched set. The temporality filter on every samples statement is a
|
||||
// semantic no-op — the matched fingerprints already come from those
|
||||
// temporalities — that engages the leading samples primary-key column.
|
||||
// Delta-temporality series stay invisible to PromQL here exactly as they are
|
||||
// in v1: the rollout gate is parity with v1, and making Delta visible is its
|
||||
// own change with its own semantics to design — a Delta stream fed to rate()
|
||||
// as-if-cumulative would be wrong, not just new.
|
||||
//
|
||||
// # Observability
|
||||
//
|
||||
// Every statement carries a log_comment with
|
||||
// code.namespace=clickhouse-prometheus-v2 and code.function.name naming the
|
||||
// call site (selectSeries, selectSamples, LabelValues, LabelNames), so this
|
||||
// provider's work is attributable in system.query_log without guessing from
|
||||
// query text.
|
||||
package clickhouseprometheusv2
|
||||
@@ -0,0 +1,191 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The lastSamplePerStep correctness argument, executed: for instant selectors, keeping
|
||||
// only the last sample of every step bucket (bucket 0 = (start, firstEval],
|
||||
// bucket i = (firstEval+(i-1)·step, firstEval+i·step]) yields exactly the
|
||||
// same instant-vector selections as the raw samples, for every evaluation
|
||||
// timestamp on the grid. The engine picks the latest sample in
|
||||
// (t-lookback, t] per evaluation timestamp t and treats a stale marker as
|
||||
// absent; both behaviors are emulated here directly.
|
||||
|
||||
type tsample struct {
|
||||
ts int64
|
||||
value float64
|
||||
stale bool
|
||||
}
|
||||
|
||||
// engineSelect emulates the engine's instant-selector resolution at
|
||||
// evaluation timestamp t over samples ordered by timestamp: the latest sample
|
||||
// in (t-lookback, t], absent when none or when it is a stale marker.
|
||||
func engineSelect(samples []tsample, t, lookbackMs int64) (tsample, bool) {
|
||||
var picked tsample
|
||||
found := false
|
||||
for _, s := range samples {
|
||||
if s.ts > t-lookbackMs && s.ts <= t {
|
||||
picked = s
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found || picked.stale {
|
||||
return tsample{}, false
|
||||
}
|
||||
return picked, true
|
||||
}
|
||||
|
||||
// lastPerStep emulates the last-sample-per-step samples query: group samples into buckets and
|
||||
// keep only the last sample of each (ties keep either; ClickHouse argMax over
|
||||
// equal keys is unspecified, so generated timestamps are unique).
|
||||
func lastPerStep(samples []tsample, firstEvalMs, stepMs int64) []tsample {
|
||||
last := make(map[int64]tsample)
|
||||
for _, s := range samples {
|
||||
var bucket int64
|
||||
if stepMs > 0 && s.ts > firstEvalMs {
|
||||
bucket = (s.ts-firstEvalMs-1)/stepMs + 1
|
||||
}
|
||||
if cur, ok := last[bucket]; !ok || s.ts > cur.ts {
|
||||
last[bucket] = s
|
||||
}
|
||||
}
|
||||
out := make([]tsample, 0, len(last))
|
||||
for _, s := range last {
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].ts < out[j].ts })
|
||||
return out
|
||||
}
|
||||
|
||||
func TestLastSamplePerStepEquivalence(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(42))
|
||||
|
||||
for caseIdx := 0; caseIdx < 2000; caseIdx++ {
|
||||
// Random query shape. Units are milliseconds but kept small so bucket
|
||||
// boundaries are hit often.
|
||||
stepMs := []int64{1, 2, 5, 7, 30, 60}[rng.Intn(6)]
|
||||
lookbackMs := []int64{1, 3, 5, 10, 45}[rng.Intn(5)]
|
||||
queryStart := int64(1000)
|
||||
numSteps := rng.Int63n(20)
|
||||
queryEnd := queryStart + numSteps*stepMs + rng.Int63n(stepMs) // grid may not divide the range
|
||||
|
||||
// Engine-derived selector window for instant selectors:
|
||||
// hints.Start = firstEval - (lookback - 1), hints.End = queryEnd.
|
||||
hintsStart := queryStart - (lookbackMs - 1)
|
||||
hintsEnd := queryEnd
|
||||
firstEval := hintsStart + lookbackMs - 1
|
||||
require.Equal(t, queryStart, firstEval)
|
||||
|
||||
// Random samples inside the fetch window [hints.Start, hints.End],
|
||||
// with unique timestamps and occasional stale markers. The sample
|
||||
// count is capped by the window size: timestamps are unique.
|
||||
windowSize := hintsEnd - hintsStart + 1
|
||||
numSamples := rng.Int63n(40)
|
||||
if numSamples > windowSize {
|
||||
numSamples = windowSize
|
||||
}
|
||||
seen := make(map[int64]bool)
|
||||
var samples []tsample
|
||||
for int64(len(samples)) < numSamples {
|
||||
ts := hintsStart + rng.Int63n(windowSize)
|
||||
if seen[ts] {
|
||||
continue
|
||||
}
|
||||
seen[ts] = true
|
||||
samples = append(samples, tsample{ts: ts, value: rng.Float64(), stale: rng.Intn(8) == 0})
|
||||
}
|
||||
sort.Slice(samples, func(i, j int) bool { return samples[i].ts < samples[j].ts })
|
||||
|
||||
reduced := lastPerStep(samples, firstEval, stepMs)
|
||||
|
||||
desc := fmt.Sprintf("case=%d step=%d lookback=%d start=%d end=%d samples=%d",
|
||||
caseIdx, stepMs, lookbackMs, queryStart, queryEnd, len(samples))
|
||||
|
||||
for evalTs := queryStart; evalTs <= queryEnd; evalTs += stepMs {
|
||||
rawPick, rawOK := engineSelect(samples, evalTs, lookbackMs)
|
||||
reducedPick, reducedOK := engineSelect(reduced, evalTs, lookbackMs)
|
||||
|
||||
require.Equal(t, rawOK, reducedOK, "%s eval=%d presence mismatch", desc, evalTs)
|
||||
if rawOK {
|
||||
require.Equal(t, rawPick, reducedPick, "%s eval=%d sample mismatch", desc, evalTs)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Instant queries (step 0) evaluate once at firstEval == hints.End; lastSamplePerStep
|
||||
// collapses to a single bucket over the whole window.
|
||||
func TestLastSamplePerStepEquivalenceInstantQuery(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(7))
|
||||
|
||||
for caseIdx := 0; caseIdx < 500; caseIdx++ {
|
||||
lookbackMs := []int64{1, 3, 5, 10, 45}[rng.Intn(5)]
|
||||
evalTs := int64(1000)
|
||||
hintsStart := evalTs - (lookbackMs - 1)
|
||||
hintsEnd := evalTs
|
||||
firstEval := hintsStart + lookbackMs - 1
|
||||
require.Equal(t, evalTs, firstEval)
|
||||
|
||||
windowSize := hintsEnd - hintsStart + 1
|
||||
numSamples := rng.Int63n(10)
|
||||
if numSamples > windowSize {
|
||||
numSamples = windowSize
|
||||
}
|
||||
seen := make(map[int64]bool)
|
||||
var samples []tsample
|
||||
for int64(len(samples)) < numSamples {
|
||||
ts := hintsStart + rng.Int63n(windowSize)
|
||||
if seen[ts] {
|
||||
continue
|
||||
}
|
||||
seen[ts] = true
|
||||
samples = append(samples, tsample{ts: ts, value: rng.Float64(), stale: rng.Intn(4) == 0})
|
||||
}
|
||||
sort.Slice(samples, func(i, j int) bool { return samples[i].ts < samples[j].ts })
|
||||
|
||||
reduced := lastPerStep(samples, firstEval, 0)
|
||||
require.LessOrEqual(t, len(reduced), 1, "instant reduction must keep at most one sample")
|
||||
|
||||
rawPick, rawOK := engineSelect(samples, evalTs, lookbackMs)
|
||||
reducedPick, reducedOK := engineSelect(reduced, evalTs, lookbackMs)
|
||||
require.Equal(t, rawOK, reducedOK, "case=%d presence mismatch", caseIdx)
|
||||
if rawOK {
|
||||
require.Equal(t, rawPick, reducedPick, "case=%d sample mismatch", caseIdx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A stale marker that is the latest sample of its bucket must shadow older
|
||||
// samples: the engine sees the marker and reports the series absent, exactly
|
||||
// as with raw samples. Pre-filtering stale rows would instead resurrect the
|
||||
// older sample.
|
||||
func TestLastSamplePerStepKeepsStaleShadowing(t *testing.T) {
|
||||
lookbackMs := int64(10)
|
||||
stepMs := int64(5)
|
||||
queryStart := int64(1000)
|
||||
|
||||
samples := []tsample{
|
||||
{ts: 998, value: 1.0}, // bucket 0
|
||||
{ts: 999, stale: true}, // bucket 0: marker shadows 998
|
||||
{ts: 1003, value: 2.0}, // bucket 1
|
||||
{ts: 1004, stale: true}, // bucket 1: marker shadows 1003
|
||||
{ts: 1008, value: 3.0, stale: false}, // bucket 2
|
||||
}
|
||||
firstEval := queryStart
|
||||
reduced := lastPerStep(samples, firstEval, stepMs)
|
||||
|
||||
for evalTs := queryStart; evalTs <= queryStart+2*stepMs; evalTs += stepMs {
|
||||
rawPick, rawOK := engineSelect(samples, evalTs, lookbackMs)
|
||||
reducedPick, reducedOK := engineSelect(reduced, evalTs, lookbackMs)
|
||||
require.Equal(t, rawOK, reducedOK, "eval=%d", evalTs)
|
||||
if rawOK {
|
||||
require.Equal(t, rawPick, reducedPick, "eval=%d", evalTs)
|
||||
}
|
||||
}
|
||||
}
|
||||
72
pkg/prometheus/clickhouseprometheusv2/provider.go
Normal file
72
pkg/prometheus/clickhouseprometheusv2/provider.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
)
|
||||
|
||||
// Provider ties the package together: its own engine and parser, and the
|
||||
// ClickHouse client behind the native storage.Querier. See the package
|
||||
// documentation for how the read path differs from v1. It is exported as a
|
||||
// concrete type — callers hold it directly, and an interface with a single
|
||||
// implementation would only hide that dependency.
|
||||
type Provider struct {
|
||||
settings factory.ScopedProviderSettings
|
||||
engine *prometheus.Engine
|
||||
parser prometheus.Parser
|
||||
client *client
|
||||
}
|
||||
|
||||
var (
|
||||
_ prometheus.Prometheus = (*Provider)(nil)
|
||||
_ prometheus.StatementCapturer = (*Provider)(nil)
|
||||
)
|
||||
|
||||
func NewFactory(telemetryStore telemetrystore.TelemetryStore) factory.ProviderFactory[prometheus.Prometheus, prometheus.Config] {
|
||||
return factory.NewProviderFactory(factory.MustNewName("clickhousev2"), func(ctx context.Context, providerSettings factory.ProviderSettings, config prometheus.Config) (prometheus.Prometheus, error) {
|
||||
return New(ctx, providerSettings, config, telemetryStore)
|
||||
})
|
||||
}
|
||||
|
||||
func New(_ context.Context, providerSettings factory.ProviderSettings, config prometheus.Config, telemetryStore telemetrystore.TelemetryStore) (*Provider, error) {
|
||||
settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2")
|
||||
|
||||
engine := prometheus.NewEngine(settings.Logger(), config)
|
||||
parser := prometheus.NewParser()
|
||||
client := newClient(settings, telemetryStore, config)
|
||||
|
||||
return &Provider{
|
||||
settings: settings,
|
||||
engine: engine,
|
||||
parser: parser,
|
||||
client: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *Provider) Engine() *prometheus.Engine {
|
||||
return p.engine
|
||||
}
|
||||
|
||||
func (p *Provider) Parser() prometheus.Parser {
|
||||
return p.parser
|
||||
}
|
||||
|
||||
func (p *Provider) Storage() storage.Queryable {
|
||||
return p
|
||||
}
|
||||
|
||||
func (p *Provider) Querier(mint, maxt int64) (storage.Querier, error) {
|
||||
return &querier{mint: mint, maxt: maxt, client: p.client}, nil
|
||||
}
|
||||
|
||||
// CapturingStorage implements prometheus.StatementCapturer: a storage that
|
||||
// records each selector's SQL without executing it, for the preview path.
|
||||
// A fresh recorder per call keeps concurrent dry-runs isolated.
|
||||
func (p *Provider) CapturingStorage() (storage.Queryable, prometheus.StatementRecorder) {
|
||||
recorder := &statementRecorder{}
|
||||
return &captureQueryable{client: p.client, recorder: recorder}, recorder
|
||||
}
|
||||
233
pkg/prometheus/clickhouseprometheusv2/querier.go
Normal file
233
pkg/prometheus/clickhouseprometheusv2/querier.go
Normal file
@@ -0,0 +1,233 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/util/annotations"
|
||||
)
|
||||
|
||||
// defaultLookbackDelta mirrors promql's default when the config leaves the
|
||||
// lookback unset; the engine and the storage must agree on it for
|
||||
// last-sample-per-step bucket anchoring.
|
||||
const defaultLookbackDelta = 5 * time.Minute
|
||||
|
||||
// querier is a native storage.Querier over ClickHouse. Unlike v1 it does not
|
||||
// round-trip through the remote-read protobuf machinery: Select builds SQL
|
||||
// directly from the matchers and hints, and the result set is assembled once
|
||||
// into compact series.
|
||||
type querier struct {
|
||||
mint, maxt int64
|
||||
client *client
|
||||
}
|
||||
|
||||
var _ storage.Querier = (*querier)(nil)
|
||||
|
||||
func (q *querier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet {
|
||||
if rawQuery, ok := rawSQLQuery(matchers); ok {
|
||||
_, end := q.window(hints)
|
||||
list, err := q.client.queryRaw(ctx, rawQuery, end)
|
||||
if err != nil {
|
||||
return storage.ErrSeriesSet(err)
|
||||
}
|
||||
if sortSeries {
|
||||
sort.Slice(list, func(i, j int) bool { return labels.Compare(list[i].lset, list[j].lset) < 0 })
|
||||
}
|
||||
return newSeriesSet(list)
|
||||
}
|
||||
|
||||
start, end := q.window(hints)
|
||||
|
||||
seriesQuery, seriesArgs, err := buildSeriesQuery(start, end, matchers)
|
||||
if err != nil {
|
||||
return storage.ErrSeriesSet(err)
|
||||
}
|
||||
lookup, err := q.client.selectSeries(ctx, seriesQuery, seriesArgs)
|
||||
if err != nil {
|
||||
return storage.ErrSeriesSet(err)
|
||||
}
|
||||
if len(lookup.fingerprints) == 0 {
|
||||
return storage.EmptySeriesSet()
|
||||
}
|
||||
|
||||
list, err := q.fetchSamples(ctx, start, end, matchers, lookup, q.lastSamplePerStepFor(ctx, hints))
|
||||
if err != nil {
|
||||
return storage.ErrSeriesSet(err)
|
||||
}
|
||||
|
||||
// Sorting doubles as duplicate-label-set detection, which the engine
|
||||
// depends on storages never emitting; the cost is on series count, not
|
||||
// samples.
|
||||
list = sortAndMerge(list)
|
||||
return newSeriesSet(list)
|
||||
}
|
||||
|
||||
// LabelValues returns the values of a label across series matching the
|
||||
// matchers within the querier window.
|
||||
func (q *querier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
if name == metricNameLabel {
|
||||
sb.Select("DISTINCT metric_name AS value")
|
||||
} else {
|
||||
sb.Select(fmt.Sprintf("DISTINCT JSONExtractString(labels, %s) AS value", sb.Var(name)))
|
||||
}
|
||||
adjustedStart, table := timeSeriesTableFor(q.mint, q.maxt)
|
||||
sb.From(fmt.Sprintf("%s.%s", databaseName, table))
|
||||
if err := applySeriesConditions(sb, adjustedStart, q.maxt, matchers); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
sb.Where("value != ''")
|
||||
if hints != nil && hints.Limit > 0 {
|
||||
sb.Limit(hints.Limit)
|
||||
}
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
values, err := q.selectStrings(ctx, "LabelValues", query, args)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
slices.Sort(values)
|
||||
return values, nil, nil
|
||||
}
|
||||
|
||||
// LabelNames returns the label names present on series matching the matchers
|
||||
// within the querier window.
|
||||
func (q *querier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("DISTINCT arrayJoin(JSONExtractKeys(labels)) AS name")
|
||||
adjustedStart, table := timeSeriesTableFor(q.mint, q.maxt)
|
||||
sb.From(fmt.Sprintf("%s.%s", databaseName, table))
|
||||
if err := applySeriesConditions(sb, adjustedStart, q.maxt, matchers); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if hints != nil && hints.Limit > 0 {
|
||||
sb.Limit(hints.Limit)
|
||||
}
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
names, err := q.selectStrings(ctx, "LabelNames", query, args)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
slices.Sort(names)
|
||||
return names, nil, nil
|
||||
}
|
||||
|
||||
func (q *querier) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// window returns the per-selector fetch window. The engine sends per-selector
|
||||
// bounds in the hints (already adjusted for offset, @, range and lookback);
|
||||
// they are always at least as tight as the querier-level mint/maxt, which
|
||||
// span the union of all selectors in the query.
|
||||
func (q *querier) window(hints *storage.SelectHints) (int64, int64) {
|
||||
if hints != nil && hints.Start != 0 && hints.End != 0 && hints.Start <= hints.End {
|
||||
return hints.Start, hints.End
|
||||
}
|
||||
return q.mint, q.maxt
|
||||
}
|
||||
|
||||
// lastSamplePerStepFor decides whether the fetch can keep only the last
|
||||
// sample per step bucket, and computes the bucket parameters. Requirements:
|
||||
// - the call site attached QueryTraits proving the query has no subquery
|
||||
// (subquery selectors evaluate at the subquery's own step, but hints
|
||||
// carry the top-level step);
|
||||
// - the selector is an instant selector (hints.Range == 0); range selectors
|
||||
// need every raw sample in the window;
|
||||
// - per-selector hints are present.
|
||||
//
|
||||
// The engine derives hints.Start for instant selectors as
|
||||
// firstEval - (lookback - 1ms), so the first evaluation timestamp is
|
||||
// recovered as hints.Start + lookback - 1ms. Bucket boundaries then coincide
|
||||
// with evaluation timestamps, which is what makes keeping only the last
|
||||
// sample per bucket lossless.
|
||||
func (q *querier) lastSamplePerStepFor(ctx context.Context, hints *storage.SelectHints) *lastSamplePerStep {
|
||||
if hints == nil || hints.Range != 0 || hints.Start <= 0 {
|
||||
return nil
|
||||
}
|
||||
traits, ok := prometheus.QueryTraitsFromContext(ctx)
|
||||
if !ok || !traits.SubqueryFree {
|
||||
return nil
|
||||
}
|
||||
firstEval := hints.Start + q.client.lookbackMs - 1
|
||||
if firstEval > hints.End {
|
||||
// Defensive: never anchor a bucket past the window.
|
||||
firstEval = hints.End
|
||||
}
|
||||
return &lastSamplePerStep{firstEvalMs: firstEval, stepMs: hints.Step}
|
||||
}
|
||||
|
||||
// fetchSamples runs the samples query for the matched series. Small sets
|
||||
// inline the fingerprints as sorted uint64 literals — literals engage the
|
||||
// samples primary key, and sorting keeps the statement deterministic for
|
||||
// logging and tests. Larger sets re-run the series predicates as a
|
||||
// shard-local IN subquery instead: inlining hundreds of thousands of
|
||||
// literals makes the statement itself the bottleneck, while the subquery is
|
||||
// a cheap primary-key scan on each shard's own series table (see
|
||||
// localTimeSeriesTable for why that is complete).
|
||||
func (q *querier) fetchSamples(ctx context.Context, start, end int64, matchers []*labels.Matcher, lookup *seriesLookup, lastPerStep *lastSamplePerStep) ([]*series, error) {
|
||||
var fingerprints []uint64
|
||||
if len(lookup.fingerprints) <= inlineFingerprintsLimit {
|
||||
fingerprints = make([]uint64, 0, len(lookup.fingerprints))
|
||||
for fp := range lookup.fingerprints {
|
||||
fingerprints = append(fingerprints, fp)
|
||||
}
|
||||
slices.Sort(fingerprints)
|
||||
}
|
||||
query, args, err := buildSamplesQuery(start, end, lookup.metricNames, fingerprints, matchers, lastPerStep)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return q.client.selectSamples(ctx, query, args, lookup)
|
||||
}
|
||||
|
||||
func (q *querier) selectStrings(ctx context.Context, fn, query string, args []any) ([]string, error) {
|
||||
ctx = q.client.withContext(ctx, fn)
|
||||
rows, err := q.client.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []string
|
||||
var v string
|
||||
for rows.Next() {
|
||||
if err := rows.Scan(&v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// rawSQLQuery detects the {job="rawsql", query="..."} escape hatch.
|
||||
func rawSQLQuery(matchers []*labels.Matcher) (string, bool) {
|
||||
if len(matchers) != 2 {
|
||||
return "", false
|
||||
}
|
||||
var hasJob bool
|
||||
var query string
|
||||
for _, m := range matchers {
|
||||
if m.Type == labels.MatchEqual && m.Name == "job" && m.Value == "rawsql" {
|
||||
hasJob = true
|
||||
}
|
||||
if m.Type == labels.MatchEqual && m.Name == "query" {
|
||||
query = m.Value
|
||||
}
|
||||
}
|
||||
if hasJob && query != "" {
|
||||
return query, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
221
pkg/prometheus/clickhouseprometheusv2/querier_test.go
Normal file
221
pkg/prometheus/clickhouseprometheusv2/querier_test.go
Normal file
@@ -0,0 +1,221 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
cmock "github.com/SigNoz/clickhouse-go-mock"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
)
|
||||
|
||||
var (
|
||||
seriesCols = []cmock.ColumnType{
|
||||
{Name: "fingerprint", Type: "UInt64"},
|
||||
{Name: "labels", Type: "String"},
|
||||
}
|
||||
samplesCols = []cmock.ColumnType{
|
||||
{Name: "fingerprint", Type: "UInt64"},
|
||||
{Name: "unix_milli", Type: "Int64"},
|
||||
{Name: "value", Type: "Float64"},
|
||||
{Name: "flags", Type: "UInt32"},
|
||||
}
|
||||
)
|
||||
|
||||
func newTestClient(t *testing.T, cfg prometheus.ClickhouseV2Config) (*client, *telemetrystoretest.Provider) {
|
||||
t.Helper()
|
||||
store := telemetrystoretest.New(telemetrystore.Config{Provider: "clickhouse"}, sqlmock.QueryMatcherRegexp)
|
||||
settings := factory.NewScopedProviderSettings(instrumentationtest.New().ToProviderSettings(), "clickhouseprometheusv2_test")
|
||||
promCfg := prometheus.Config{ClickhouseV2: cfg}
|
||||
return newClient(settings, store, promCfg), store
|
||||
}
|
||||
|
||||
func testMatchers(t *testing.T) []*labels.Matcher {
|
||||
t.Helper()
|
||||
return []*labels.Matcher{
|
||||
mustMatcher(t, labels.MatchEqual, "__name__", "cpu_usage"),
|
||||
mustMatcher(t, labels.MatchEqual, "job", "api"),
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuerierSelectRawPath(t *testing.T) {
|
||||
c, store := newTestClient(t, prometheus.ClickhouseV2Config{})
|
||||
q := &querier{mint: 1000, maxt: 2000, client: c}
|
||||
|
||||
store.Mock().ExpectQuery("SELECT fingerprint, any\\(labels\\)").WithArgs("cpu_usage", int64(0), int64(2000), "job", "api").WillReturnRows(cmock.NewRows(seriesCols, [][]any{
|
||||
{uint64(42), `{"__name__":"cpu_usage","job":"api","instance":"a"}`},
|
||||
{uint64(7), `{"__name__":"cpu_usage","job":"api","instance":"b"}`},
|
||||
}))
|
||||
// Inline fingerprints (sorted), raw samples: no traits in ctx -> no
|
||||
// last-sample-per-step reduction.
|
||||
store.Mock().ExpectQuery("SELECT fingerprint, unix_milli, value, flags FROM signoz_metrics.distributed_samples_v4 WHERE metric_name = \\? AND temporality IN \\['Cumulative', 'Unspecified'\\] AND fingerprint IN \\(7, 42\\)").
|
||||
WithArgs("cpu_usage", int64(1000), int64(2000)).
|
||||
WillReturnRows(cmock.NewRows(samplesCols, [][]any{
|
||||
{uint64(7), int64(1100), 1.5, uint32(0)},
|
||||
{uint64(7), int64(1200), 2.5, uint32(0)},
|
||||
{uint64(42), int64(1100), 3.5, uint32(1)}, // stale marker
|
||||
}))
|
||||
|
||||
hints := &storage.SelectHints{Start: 1000, End: 2000, Step: 60_000}
|
||||
set := q.Select(context.Background(), false, hints, testMatchers(t)...)
|
||||
|
||||
var got []*series
|
||||
for set.Next() {
|
||||
got = append(got, set.At().(*series))
|
||||
}
|
||||
require.NoError(t, set.Err())
|
||||
require.Len(t, got, 2)
|
||||
|
||||
// Sorted by labels: instance=a (fp 42) before instance=b (fp 7).
|
||||
assert.Equal(t, "a", got[0].lset.Get("instance"))
|
||||
require.Len(t, got[0].ts, 1)
|
||||
assert.True(t, got[0].vs[0] != got[0].vs[0], "stale marker must be NaN") //nolint:testifylint
|
||||
|
||||
assert.Equal(t, "b", got[1].lset.Get("instance"))
|
||||
assert.Equal(t, []int64{1100, 1200}, got[1].ts)
|
||||
assert.Equal(t, []float64{1.5, 2.5}, got[1].vs)
|
||||
|
||||
// No fingerprint label injected.
|
||||
assert.Empty(t, got[0].lset.Get("fingerprint"))
|
||||
}
|
||||
|
||||
// Wrong gating silently corrupts range functions (a rate over reduced
|
||||
// samples loses points), so the decision logic is pinned here even though
|
||||
// the helper is unexported: the integration suite would catch it too, but
|
||||
// with far worse failure locality.
|
||||
func TestLastSamplePerStepFor(t *testing.T) {
|
||||
c, _ := newTestClient(t, prometheus.ClickhouseV2Config{})
|
||||
q := &querier{mint: 0, maxt: 2000, client: c}
|
||||
traitsCtx := prometheus.NewContextWithQueryTraits(context.Background(), prometheus.QueryTraits{SubqueryFree: true})
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
ctx context.Context
|
||||
hints *storage.SelectHints
|
||||
want *lastSamplePerStep
|
||||
}{
|
||||
{"no traits in context stays raw", context.Background(), &storage.SelectHints{Start: 1000, End: 2000, Step: 60_000}, nil},
|
||||
{"subquery in the query stays raw", prometheus.NewContextWithQueryTraits(context.Background(), prometheus.QueryTraits{SubqueryFree: false}), &storage.SelectHints{Start: 1000, End: 2000, Step: 60_000}, nil},
|
||||
{"range selector stays raw", traitsCtx, &storage.SelectHints{Start: 1000, End: 2000, Step: 60_000, Range: 300_000}, nil},
|
||||
{"instant selector reduces, anchored at first eval", traitsCtx, &storage.SelectHints{Start: 1000, End: 2_000_000, Step: 60_000}, &lastSamplePerStep{firstEvalMs: 1000 + c.lookbackMs - 1, stepMs: 60_000}},
|
||||
{"anchor never passes the window end", traitsCtx, &storage.SelectHints{Start: 1000, End: 2000, Step: 60_000}, &lastSamplePerStep{firstEvalMs: 2000, stepMs: 60_000}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, q.lastSamplePerStepFor(tt.ctx, tt.hints))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuerierSelectSeriesBudget(t *testing.T) {
|
||||
c, store := newTestClient(t, prometheus.ClickhouseV2Config{MaxFetchedSeries: 1})
|
||||
q := &querier{mint: 1000, maxt: 2000, client: c}
|
||||
|
||||
store.Mock().ExpectQuery("SELECT fingerprint, any\\(labels\\)").WithArgs("cpu_usage", int64(0), int64(2000), "job", "api").WillReturnRows(cmock.NewRows(seriesCols, [][]any{
|
||||
{uint64(1), `{"__name__":"cpu_usage","instance":"a"}`},
|
||||
{uint64(2), `{"__name__":"cpu_usage","instance":"b"}`},
|
||||
}))
|
||||
|
||||
set := q.Select(context.Background(), false, &storage.SelectHints{Start: 1000, End: 2000}, testMatchers(t)...)
|
||||
assert.False(t, set.Next())
|
||||
require.Error(t, set.Err())
|
||||
assert.True(t, errors.Ast(set.Err(), errors.TypeInvalidInput), "budget error must be typed invalid input, got %v", set.Err())
|
||||
}
|
||||
|
||||
func TestQuerierSelectSamplesBudget(t *testing.T) {
|
||||
c, store := newTestClient(t, prometheus.ClickhouseV2Config{MaxFetchedSamples: 2})
|
||||
q := &querier{mint: 1000, maxt: 2000, client: c}
|
||||
|
||||
store.Mock().ExpectQuery("SELECT fingerprint, any\\(labels\\)").WithArgs("cpu_usage", int64(0), int64(2000)).WillReturnRows(cmock.NewRows(seriesCols, [][]any{
|
||||
{uint64(7), `{"__name__":"cpu_usage"}`},
|
||||
}))
|
||||
store.Mock().ExpectQuery("SELECT fingerprint, unix_milli, value, flags").
|
||||
WithArgs("cpu_usage", int64(1000), int64(2000)).
|
||||
WillReturnRows(cmock.NewRows(samplesCols, [][]any{
|
||||
{uint64(7), int64(1100), 1.0, uint32(0)},
|
||||
{uint64(7), int64(1200), 2.0, uint32(0)},
|
||||
{uint64(7), int64(1300), 3.0, uint32(0)},
|
||||
}))
|
||||
|
||||
set := q.Select(context.Background(), false, &storage.SelectHints{Start: 1000, End: 2000},
|
||||
mustMatcher(t, labels.MatchEqual, "__name__", "cpu_usage"))
|
||||
assert.False(t, set.Next())
|
||||
require.Error(t, set.Err())
|
||||
assert.True(t, errors.Ast(set.Err(), errors.TypeInvalidInput))
|
||||
}
|
||||
|
||||
func TestQuerierSelectSubqueryFilterOverInlineLimit(t *testing.T) {
|
||||
c, store := newTestClient(t, prometheus.ClickhouseV2Config{})
|
||||
q := &querier{mint: 1000, maxt: 2000, client: c}
|
||||
|
||||
seriesRows := make([][]any, inlineFingerprintsLimit+1)
|
||||
for i := range seriesRows {
|
||||
seriesRows[i] = []any{uint64(i + 1), fmt.Sprintf(`{"__name__":"cpu_usage","instance":"i%d"}`, i)}
|
||||
}
|
||||
store.Mock().ExpectQuery("SELECT fingerprint, any\\(labels\\)").WithArgs("cpu_usage", int64(0), int64(2000), "job", "api").WillReturnRows(cmock.NewRows(seriesCols, seriesRows))
|
||||
// The over-limit samples query embeds the semi-join against the
|
||||
// shard-local series table (fingerprint co-locality), not a GLOBAL
|
||||
// broadcast; args follow placeholder order — samples metric name, then
|
||||
// the semi-join's series predicates, then the samples window bounds.
|
||||
store.Mock().ExpectQuery("fingerprint IN \\(SELECT fingerprint FROM signoz_metrics\\.time_series_v4").
|
||||
WithArgs("cpu_usage", "cpu_usage", int64(0), int64(2000), "job", "api", int64(1000), int64(2000)).
|
||||
WillReturnRows(cmock.NewRows(samplesCols, [][]any{}))
|
||||
|
||||
set := q.Select(context.Background(), false, &storage.SelectHints{Start: 1000, End: 2000}, testMatchers(t)...)
|
||||
assert.False(t, set.Next())
|
||||
require.NoError(t, set.Err())
|
||||
}
|
||||
|
||||
func TestQuerierSelectRawSQLPassthrough(t *testing.T) {
|
||||
c, store := newTestClient(t, prometheus.ClickhouseV2Config{})
|
||||
q := &querier{mint: 1000, maxt: 2000, client: c}
|
||||
|
||||
rawCols := []cmock.ColumnType{
|
||||
{Name: "le", Type: "String"},
|
||||
{Name: "value", Type: "Float64"},
|
||||
}
|
||||
store.Mock().ExpectQuery("SELECT le, avg\\(v\\) AS value FROM t").WillReturnRows(cmock.NewRows(rawCols, [][]any{
|
||||
{"0.5", 12.5},
|
||||
}))
|
||||
|
||||
set := q.Select(context.Background(), false, &storage.SelectHints{Start: 1000, End: 2000},
|
||||
mustMatcher(t, labels.MatchEqual, "job", "rawsql"),
|
||||
mustMatcher(t, labels.MatchEqual, "query", "SELECT le, avg(v) AS value FROM t"),
|
||||
)
|
||||
|
||||
require.True(t, set.Next())
|
||||
s := set.At()
|
||||
assert.Equal(t, "0.5", s.Labels().Get("le"))
|
||||
it := s.Iterator(nil)
|
||||
require.NotNil(t, it)
|
||||
_, v := func() (int64, float64) { it.Next(); return it.At() }()
|
||||
assert.Equal(t, 12.5, v)
|
||||
assert.False(t, set.Next())
|
||||
}
|
||||
|
||||
func TestCaptureQuerierRecordsWithoutExecuting(t *testing.T) {
|
||||
c, _ := newTestClient(t, prometheus.ClickhouseV2Config{})
|
||||
recorder := &statementRecorder{}
|
||||
cq := &captureQuerier{querier: querier{mint: 1000, maxt: 2000, client: c}, recorder: recorder}
|
||||
|
||||
ctx := prometheus.NewContextWithQueryTraits(context.Background(), prometheus.QueryTraits{SubqueryFree: true})
|
||||
set := cq.Select(ctx, false, &storage.SelectHints{Start: 1000, End: 2000, Step: 60_000}, testMatchers(t)...)
|
||||
assert.False(t, set.Next())
|
||||
require.NoError(t, set.Err())
|
||||
|
||||
statements := recorder.Statements()
|
||||
require.Len(t, statements, 1)
|
||||
assert.Contains(t, statements[0].Query, "IN (SELECT fingerprint FROM signoz_metrics.time_series_v4")
|
||||
assert.Contains(t, statements[0].Query, "argMax(value, unix_milli)")
|
||||
}
|
||||
184
pkg/prometheus/clickhouseprometheusv2/seriesset.go
Normal file
184
pkg/prometheus/clickhouseprometheusv2/seriesset.go
Normal file
@@ -0,0 +1,184 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/prometheus/prometheus/util/annotations"
|
||||
)
|
||||
|
||||
// series is one time series with samples stored as parallel slices, ordered
|
||||
// by timestamp. The compact layout avoids per-sample allocations and keeps
|
||||
// iteration cache friendly.
|
||||
type series struct {
|
||||
lset labels.Labels
|
||||
ts []int64
|
||||
vs []float64
|
||||
}
|
||||
|
||||
var _ storage.Series = (*series)(nil)
|
||||
|
||||
func (s *series) Labels() labels.Labels {
|
||||
return s.lset
|
||||
}
|
||||
|
||||
func (s *series) Iterator(it chunkenc.Iterator) chunkenc.Iterator {
|
||||
if fit, ok := it.(*floatIterator); ok {
|
||||
fit.reset(s)
|
||||
return fit
|
||||
}
|
||||
fit := &floatIterator{}
|
||||
fit.reset(s)
|
||||
return fit
|
||||
}
|
||||
|
||||
// floatIterator implements chunkenc.Iterator over a series' sample slices.
|
||||
type floatIterator struct {
|
||||
s *series
|
||||
i int
|
||||
}
|
||||
|
||||
var _ chunkenc.Iterator = (*floatIterator)(nil)
|
||||
|
||||
func (it *floatIterator) reset(s *series) {
|
||||
it.s = s
|
||||
it.i = -1
|
||||
}
|
||||
|
||||
func (it *floatIterator) Next() chunkenc.ValueType {
|
||||
it.i++
|
||||
if it.i >= len(it.s.ts) {
|
||||
return chunkenc.ValNone
|
||||
}
|
||||
return chunkenc.ValFloat
|
||||
}
|
||||
|
||||
func (it *floatIterator) Seek(t int64) chunkenc.ValueType { //nolint:govet // stdmethods flags io.Seeker; this is chunkenc.Iterator's Seek
|
||||
if it.i < 0 {
|
||||
it.i = 0
|
||||
}
|
||||
if it.i >= len(it.s.ts) {
|
||||
return chunkenc.ValNone
|
||||
}
|
||||
// The current position, once valid, must not move backwards.
|
||||
if it.s.ts[it.i] >= t {
|
||||
return chunkenc.ValFloat
|
||||
}
|
||||
it.i += sort.Search(len(it.s.ts)-it.i, func(j int) bool {
|
||||
return it.s.ts[it.i+j] >= t
|
||||
})
|
||||
if it.i >= len(it.s.ts) {
|
||||
return chunkenc.ValNone
|
||||
}
|
||||
return chunkenc.ValFloat
|
||||
}
|
||||
|
||||
func (it *floatIterator) At() (int64, float64) {
|
||||
return it.s.ts[it.i], it.s.vs[it.i]
|
||||
}
|
||||
|
||||
func (it *floatIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (it *floatIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (it *floatIterator) AtT() int64 {
|
||||
return it.s.ts[it.i]
|
||||
}
|
||||
|
||||
// AtST returns the current start timestamp; not tracked by this storage.
|
||||
func (it *floatIterator) AtST() int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (it *floatIterator) Err() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// seriesSet iterates a fully materialized, label-sorted list of series.
|
||||
type seriesSet struct {
|
||||
series []*series
|
||||
i int
|
||||
}
|
||||
|
||||
var _ storage.SeriesSet = (*seriesSet)(nil)
|
||||
|
||||
func newSeriesSet(list []*series) *seriesSet {
|
||||
return &seriesSet{series: list, i: -1}
|
||||
}
|
||||
|
||||
func (s *seriesSet) Next() bool {
|
||||
s.i++
|
||||
return s.i < len(s.series)
|
||||
}
|
||||
|
||||
func (s *seriesSet) At() storage.Series {
|
||||
return s.series[s.i]
|
||||
}
|
||||
|
||||
func (s *seriesSet) Err() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *seriesSet) Warnings() annotations.Annotations {
|
||||
return nil
|
||||
}
|
||||
|
||||
// sortAndMerge orders series by label set and merges series whose label sets
|
||||
// are identical. Distinct fingerprints can carry identical label sets (e.g.
|
||||
// series differing only in a non-label dimension); Prometheus storages never
|
||||
// expose duplicate label sets to the engine, so merge their samples by
|
||||
// timestamp, keeping the first sample on ties.
|
||||
func sortAndMerge(list []*series) []*series {
|
||||
if len(list) < 2 {
|
||||
return list
|
||||
}
|
||||
sort.Slice(list, func(i, j int) bool {
|
||||
return labels.Compare(list[i].lset, list[j].lset) < 0
|
||||
})
|
||||
out := list[:1]
|
||||
for _, s := range list[1:] {
|
||||
last := out[len(out)-1]
|
||||
if labels.Compare(last.lset, s.lset) != 0 {
|
||||
out = append(out, s)
|
||||
continue
|
||||
}
|
||||
merged := mergeSamples(last, s)
|
||||
out[len(out)-1] = merged
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mergeSamples(a, b *series) *series {
|
||||
ts := make([]int64, 0, len(a.ts)+len(b.ts))
|
||||
vs := make([]float64, 0, len(a.ts)+len(b.ts))
|
||||
i, j := 0, 0
|
||||
for i < len(a.ts) && j < len(b.ts) {
|
||||
switch {
|
||||
case a.ts[i] < b.ts[j]:
|
||||
ts = append(ts, a.ts[i])
|
||||
vs = append(vs, a.vs[i])
|
||||
i++
|
||||
case a.ts[i] > b.ts[j]:
|
||||
ts = append(ts, b.ts[j])
|
||||
vs = append(vs, b.vs[j])
|
||||
j++
|
||||
default:
|
||||
ts = append(ts, a.ts[i])
|
||||
vs = append(vs, a.vs[i])
|
||||
i++
|
||||
j++
|
||||
}
|
||||
}
|
||||
ts = append(ts, a.ts[i:]...)
|
||||
vs = append(vs, a.vs[i:]...)
|
||||
ts = append(ts, b.ts[j:]...)
|
||||
vs = append(vs, b.vs[j:]...)
|
||||
return &series{lset: a.lset, ts: ts, vs: vs}
|
||||
}
|
||||
201
pkg/prometheus/clickhouseprometheusv2/sql.go
Normal file
201
pkg/prometheus/clickhouseprometheusv2/sql.go
Normal file
@@ -0,0 +1,201 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
)
|
||||
|
||||
// inlineFingerprintsLimit is the largest matched-series count inlined into
|
||||
// the samples query as literals. Literals engage the samples primary key and
|
||||
// avoid a second series-table scan; past a few thousand the statement itself
|
||||
// becomes the cost, and the shard-local subquery filter wins. Not
|
||||
// configurable: the crossover depends on statement parsing, not on any
|
||||
// property of a deployment an operator could know better.
|
||||
const inlineFingerprintsLimit = 5_000
|
||||
|
||||
// buildSeriesQuery renders the series lookup: one row per matched fingerprint
|
||||
// with its labels.
|
||||
func buildSeriesQuery(start, end int64, matchers []*labels.Matcher) (string, []any, error) {
|
||||
adjustedStart, table := timeSeriesTableFor(start, end)
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("fingerprint", "any(labels)")
|
||||
sb.From(fmt.Sprintf("%s.%s", databaseName, table))
|
||||
if err := applySeriesConditions(sb, adjustedStart, end, matchers); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
sb.GroupBy("fingerprint")
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
return query, args, nil
|
||||
}
|
||||
|
||||
// buildSamplesQuery renders the samples fetch for the series selected by the
|
||||
// series lookup. Small matched sets pass inlineFingerprints — sorted uint64
|
||||
// literals that engage the samples primary key; nil means the set exceeded
|
||||
// the inline limit, and the filter becomes a semi-join re-running the series
|
||||
// predicates against the shard-local series table (complete by fingerprint
|
||||
// co-locality, see localTimeSeriesTable; a GLOBAL broadcast of the matched
|
||||
// set would ship it to every shard instead). metricNames narrows the
|
||||
// primary-key scan; when the selector had no __name__ equality, the names
|
||||
// observed on the matched series are used. A non-nil lastPerStep groups to
|
||||
// one (the last) sample per step bucket.
|
||||
func buildSamplesQuery(start, end int64, metricNames []string, inlineFingerprints []uint64, matchers []*labels.Matcher, lastPerStep *lastSamplePerStep) (string, []any, error) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
if lastPerStep != nil {
|
||||
// Aliases must not shadow source columns: ClickHouse resolves aliases
|
||||
// in WHERE too, and "max(unix_milli) AS unix_milli" would put an
|
||||
// aggregate into the WHERE clause (error 184).
|
||||
sb.Select("fingerprint", "max(unix_milli) AS ts", "argMax(value, unix_milli) AS val", "argMax(flags, unix_milli) AS fl")
|
||||
} else {
|
||||
sb.Select("fingerprint", "unix_milli", "value", "flags")
|
||||
}
|
||||
sb.From(fmt.Sprintf("%s.%s", databaseName, distributedSamplesV4))
|
||||
|
||||
switch len(metricNames) {
|
||||
case 0:
|
||||
// No name constraint derivable; correct but unable to use the
|
||||
// metric_name primary-key prefix.
|
||||
case 1:
|
||||
sb.Where(sb.EQ("metric_name", metricNames[0]))
|
||||
default:
|
||||
sb.Where(sb.In("metric_name", sqlbuilder.List(metricNames)))
|
||||
}
|
||||
// temporality precedes metric_name in the samples primary key; the
|
||||
// fingerprints already come from these temporalities, so this only helps
|
||||
// granule pruning.
|
||||
sb.Where("temporality IN ['Cumulative', 'Unspecified']")
|
||||
if inlineFingerprints != nil {
|
||||
sb.Where("fingerprint " + inlineFingerprintFilter(inlineFingerprints))
|
||||
} else {
|
||||
sub := sqlbuilder.NewSelectBuilder()
|
||||
sub.Select("fingerprint")
|
||||
adjustedStart, table := timeSeriesTableFor(start, end)
|
||||
sub.From(fmt.Sprintf("%s.%s", databaseName, localTimeSeriesTable(table)))
|
||||
if err := applySeriesConditions(sub, adjustedStart, end, matchers); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
sb.Where(sb.In("fingerprint", sub))
|
||||
}
|
||||
sb.Where(sb.GTE("unix_milli", start), sb.LTE("unix_milli", end))
|
||||
|
||||
if lastPerStep != nil {
|
||||
sb.GroupBy("fingerprint")
|
||||
if expr := lastPerStep.bucketExpr(); expr != "" {
|
||||
sb.GroupBy(expr)
|
||||
}
|
||||
sb.OrderBy("fingerprint", "ts")
|
||||
} else {
|
||||
sb.OrderBy("fingerprint", "unix_milli")
|
||||
}
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
return query, args, nil
|
||||
}
|
||||
|
||||
// applySeriesConditions adds the WHERE conditions of a series table scan for
|
||||
// the given matchers and window. __name__ matchers translate to the
|
||||
// metric_name column (all four matcher types — the v1 client silently
|
||||
// returned nothing for regex metric names); every other matcher translates
|
||||
// to a JSONExtractString condition on the labels column. An equality matcher
|
||||
// against "" matches series without the label, mirroring PromQL, because
|
||||
// JSONExtractString returns "" for missing keys. Regexes are anchored:
|
||||
// PromQL matchers match the whole value, while ClickHouse match() searches
|
||||
// for a partial match — without anchoring, =~"api" would also select
|
||||
// "x-api-y".
|
||||
func applySeriesConditions(sb *sqlbuilder.SelectBuilder, start, end int64, matchers []*labels.Matcher) error {
|
||||
for _, m := range matchers {
|
||||
if m.Name != metricNameLabel {
|
||||
continue
|
||||
}
|
||||
switch m.Type {
|
||||
case labels.MatchEqual:
|
||||
sb.Where(sb.EQ("metric_name", m.Value))
|
||||
case labels.MatchNotEqual:
|
||||
sb.Where(sb.NE("metric_name", m.Value))
|
||||
case labels.MatchRegexp:
|
||||
sb.Where(fmt.Sprintf("match(metric_name, %s)", sb.Var(anchorRegex(m.Value))))
|
||||
case labels.MatchNotRegexp:
|
||||
sb.Where(fmt.Sprintf("NOT match(metric_name, %s)", sb.Var(anchorRegex(m.Value))))
|
||||
default:
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported matcher type %q for __name__", m.Type)
|
||||
}
|
||||
}
|
||||
|
||||
sb.Where("temporality IN ['Cumulative', 'Unspecified']")
|
||||
// Inclusive upper bound: registration rows are hour-floored (and 6h/1d/1w
|
||||
// for the rollup tables) by the exporter, so a series first registered in
|
||||
// the bucket starting exactly at `end` would otherwise be invisible while
|
||||
// its samples (<= end) are in range.
|
||||
sb.Where(sb.GTE("unix_milli", start), sb.LTE("unix_milli", end))
|
||||
|
||||
for _, m := range matchers {
|
||||
if m.Name == metricNameLabel {
|
||||
continue
|
||||
}
|
||||
switch m.Type {
|
||||
case labels.MatchEqual:
|
||||
sb.Where(fmt.Sprintf("JSONExtractString(labels, %s) = %s", sb.Var(m.Name), sb.Var(m.Value)))
|
||||
case labels.MatchNotEqual:
|
||||
sb.Where(fmt.Sprintf("JSONExtractString(labels, %s) != %s", sb.Var(m.Name), sb.Var(m.Value)))
|
||||
case labels.MatchRegexp:
|
||||
sb.Where(fmt.Sprintf("match(JSONExtractString(labels, %s), %s)", sb.Var(m.Name), sb.Var(anchorRegex(m.Value))))
|
||||
case labels.MatchNotRegexp:
|
||||
sb.Where(fmt.Sprintf("NOT match(JSONExtractString(labels, %s), %s)", sb.Var(m.Name), sb.Var(anchorRegex(m.Value))))
|
||||
default:
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported matcher type %q", m.Type)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// anchorRegex turns a PromQL regex into its fully-anchored form (see
|
||||
// applySeriesConditions).
|
||||
func anchorRegex(v string) string {
|
||||
return "^(?:" + v + ")$"
|
||||
}
|
||||
|
||||
// inlineFingerprintFilter renders "IN (fp1, fp2, ...)" with literal uint64s.
|
||||
func inlineFingerprintFilter(fingerprints []uint64) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(fingerprints)*21 + 8)
|
||||
b.WriteString("IN (")
|
||||
for i, fp := range fingerprints {
|
||||
if i > 0 {
|
||||
b.WriteString(", ")
|
||||
}
|
||||
b.WriteString(strconv.FormatUint(fp, 10))
|
||||
}
|
||||
b.WriteString(")")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// lastSamplePerStep reduces an instant-selector fetch to the last sample of
|
||||
// each step bucket. Buckets are anchored at the selector's first evaluation
|
||||
// timestamp so that every bucket boundary coincides with an evaluation
|
||||
// timestamp: bucket 0 is (start, firstEval] (the initial lookback window)
|
||||
// and bucket i is (firstEval+(i-1)·step, firstEval+i·step]. Keeping only the
|
||||
// last sample per bucket is lossless: the engine resolves each evaluation
|
||||
// timestamp t to the latest sample in (t-lookback, t], and a non-final
|
||||
// sample of a bucket can never be that latest sample for any t on the
|
||||
// evaluation grid. Real timestamps are preserved, so the engine's own
|
||||
// lookback and staleness handling remain exact.
|
||||
type lastSamplePerStep struct {
|
||||
firstEvalMs int64
|
||||
stepMs int64
|
||||
}
|
||||
|
||||
func (t *lastSamplePerStep) bucketExpr() string {
|
||||
if t.stepMs <= 0 {
|
||||
// Instant query: a single evaluation at firstEval; one bucket.
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"if(unix_milli <= %d, 0, intDiv(unix_milli - %d - 1, %d) + 1)",
|
||||
t.firstEvalMs, t.firstEvalMs, t.stepMs,
|
||||
)
|
||||
}
|
||||
148
pkg/prometheus/clickhouseprometheusv2/sql_test.go
Normal file
148
pkg/prometheus/clickhouseprometheusv2/sql_test.go
Normal file
@@ -0,0 +1,148 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func mustMatcher(t *testing.T, mt labels.MatchType, name, value string) *labels.Matcher {
|
||||
t.Helper()
|
||||
m, err := labels.NewMatcher(mt, name, value)
|
||||
require.NoError(t, err)
|
||||
return m
|
||||
}
|
||||
|
||||
func TestTimeSeriesTableFor(t *testing.T) {
|
||||
base := time.Date(2026, 7, 10, 3, 27, 0, 0, time.UTC).UnixMilli()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
span time.Duration
|
||||
wantTable string
|
||||
roundTo time.Duration
|
||||
}{
|
||||
{"under 6h uses hourly table", 2 * time.Hour, distributedTimeSeriesV4, time.Hour},
|
||||
{"under 1d uses 6h table", 12 * time.Hour, distributedTimeSeriesV46hrs, 6 * time.Hour},
|
||||
{"under 1w uses 1d table", 3 * 24 * time.Hour, distributedTimeSeriesV41day, 24 * time.Hour},
|
||||
{"over 1w uses 1w table", 10 * 24 * time.Hour, distributedTimeSeriesV41week, 7 * 24 * time.Hour},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
start, table := timeSeriesTableFor(base, base+tt.span.Milliseconds())
|
||||
assert.Equal(t, tt.wantTable, table)
|
||||
assert.Zero(t, start%tt.roundTo.Milliseconds())
|
||||
assert.LessOrEqual(t, start, base)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSeriesQuery(t *testing.T) {
|
||||
start := int64(1_700_000_000_000)
|
||||
end := start + time.Hour.Milliseconds()
|
||||
// The series table window rounds down to the table's bucket boundary.
|
||||
adjustedStart := start - (start % time.Hour.Milliseconds())
|
||||
|
||||
t.Run("equality name and label matchers", func(t *testing.T) {
|
||||
query, args, err := buildSeriesQuery(start, end, []*labels.Matcher{
|
||||
mustMatcher(t, labels.MatchEqual, "__name__", "http_requests_total"),
|
||||
mustMatcher(t, labels.MatchEqual, "job", "api"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t,
|
||||
"SELECT fingerprint, any(labels) FROM signoz_metrics.distributed_time_series_v4 WHERE metric_name = ? AND temporality IN ['Cumulative', 'Unspecified'] AND unix_milli >= ? AND unix_milli <= ? AND JSONExtractString(labels, ?) = ? GROUP BY fingerprint",
|
||||
query,
|
||||
)
|
||||
assert.Equal(t, []any{"http_requests_total", adjustedStart, end, "job", "api"}, args)
|
||||
})
|
||||
|
||||
t.Run("regex matchers are anchored", func(t *testing.T) {
|
||||
_, args, err := buildSeriesQuery(start, end, []*labels.Matcher{
|
||||
mustMatcher(t, labels.MatchEqual, "__name__", "up"),
|
||||
mustMatcher(t, labels.MatchRegexp, "instance", "prod.*"),
|
||||
mustMatcher(t, labels.MatchNotRegexp, "env", "dev|test"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []any{"up", adjustedStart, end, "instance", "^(?:prod.*)$", "env", "^(?:dev|test)$"}, args)
|
||||
})
|
||||
|
||||
t.Run("regex name matcher uses metric_name column", func(t *testing.T) {
|
||||
query, args, err := buildSeriesQuery(start, end, []*labels.Matcher{
|
||||
mustMatcher(t, labels.MatchRegexp, "__name__", "node_cpu.*|node_memory.*"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, query, "match(metric_name, ?)")
|
||||
assert.NotContains(t, query, "JSONExtractString")
|
||||
assert.Equal(t, []any{"^(?:node_cpu.*|node_memory.*)$", adjustedStart, end}, args)
|
||||
})
|
||||
|
||||
t.Run("no name matcher omits metric_name condition", func(t *testing.T) {
|
||||
query, _, err := buildSeriesQuery(start, end, []*labels.Matcher{
|
||||
mustMatcher(t, labels.MatchEqual, "job", "api"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, query, "metric_name")
|
||||
})
|
||||
}
|
||||
|
||||
func TestBuildSamplesQuery(t *testing.T) {
|
||||
start := int64(1_700_000_000_000)
|
||||
end := start + time.Hour.Milliseconds()
|
||||
adjustedStart := start - (start % time.Hour.Milliseconds())
|
||||
matchers := []*labels.Matcher{
|
||||
mustMatcher(t, labels.MatchEqual, "__name__", "up"),
|
||||
mustMatcher(t, labels.MatchEqual, "job", "api"),
|
||||
}
|
||||
|
||||
t.Run("raw with inline fingerprints", func(t *testing.T) {
|
||||
query, args, err := buildSamplesQuery(start, end, []string{"up"}, []uint64{7, 42}, matchers, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t,
|
||||
"SELECT fingerprint, unix_milli, value, flags FROM signoz_metrics.distributed_samples_v4 WHERE metric_name = ? AND temporality IN ['Cumulative', 'Unspecified'] AND fingerprint IN (7, 42) AND unix_milli >= ? AND unix_milli <= ? ORDER BY fingerprint, unix_milli",
|
||||
query,
|
||||
)
|
||||
assert.Equal(t, []any{"up", start, end}, args)
|
||||
})
|
||||
|
||||
t.Run("last-sample-per-step groups by step bucket anchored at first eval", func(t *testing.T) {
|
||||
lastPerStep := &lastSamplePerStep{firstEvalMs: start + 299_999, stepMs: 60_000}
|
||||
query, _, err := buildSamplesQuery(start, end, []string{"up"}, []uint64{7}, matchers, lastPerStep)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, query, "argMax(value, unix_milli) AS val")
|
||||
assert.Contains(t, query, "argMax(flags, unix_milli) AS fl")
|
||||
assert.Contains(t, query, "GROUP BY fingerprint, if(unix_milli <= 1700000299999, 0, intDiv(unix_milli - 1700000299999 - 1, 60000) + 1)")
|
||||
assert.Contains(t, query, "ORDER BY fingerprint, ts")
|
||||
// Aliases must not shadow the source columns referenced in WHERE.
|
||||
assert.NotContains(t, query, "AS unix_milli")
|
||||
assert.NotContains(t, query, "AS value")
|
||||
assert.NotContains(t, query, "AS flags")
|
||||
})
|
||||
|
||||
t.Run("instant query keeps one bucket", func(t *testing.T) {
|
||||
lastPerStep := &lastSamplePerStep{firstEvalMs: end, stepMs: 0}
|
||||
query, _, err := buildSamplesQuery(start, end, []string{"up"}, []uint64{7}, matchers, lastPerStep)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, query, "GROUP BY fingerprint ORDER BY fingerprint, ts")
|
||||
assert.NotContains(t, query, "intDiv")
|
||||
})
|
||||
|
||||
t.Run("over-limit set becomes a shard-local semi-join", func(t *testing.T) {
|
||||
query, args, err := buildSamplesQuery(start, end, []string{"up"}, nil, matchers, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, query, "fingerprint IN (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE ")
|
||||
assert.NotContains(t, query, "GLOBAL IN")
|
||||
// Args follow placeholder order: samples metric name, the semi-join's
|
||||
// series predicates, then the samples window bounds.
|
||||
assert.Equal(t, []any{"up", "up", adjustedStart, end, "job", "api", start, end}, args)
|
||||
})
|
||||
|
||||
t.Run("multiple metric names from regex selector", func(t *testing.T) {
|
||||
query, args, err := buildSamplesQuery(start, end, []string{"node_cpu", "node_memory"}, []uint64{7}, matchers, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, query, "metric_name IN (?, ?)")
|
||||
assert.Equal(t, []any{"node_cpu", "node_memory", start, end}, args)
|
||||
})
|
||||
}
|
||||
65
pkg/prometheus/clickhouseprometheusv2/tables.go
Normal file
65
pkg/prometheus/clickhouseprometheusv2/tables.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import "time"
|
||||
|
||||
const (
|
||||
// metricNameLabel is the reserved PromQL label holding the metric name.
|
||||
metricNameLabel string = "__name__"
|
||||
|
||||
databaseName string = "signoz_metrics"
|
||||
distributedTimeSeriesV4 string = "distributed_time_series_v4"
|
||||
distributedTimeSeriesV46hrs string = "distributed_time_series_v4_6hrs"
|
||||
distributedTimeSeriesV41day string = "distributed_time_series_v4_1day"
|
||||
distributedTimeSeriesV41week string = "distributed_time_series_v4_1week"
|
||||
distributedSamplesV4 string = "distributed_samples_v4"
|
||||
|
||||
localTimeSeriesV4 string = "time_series_v4"
|
||||
localTimeSeriesV46hrs string = "time_series_v4_6hrs"
|
||||
localTimeSeriesV41day string = "time_series_v4_1day"
|
||||
localTimeSeriesV41week string = "time_series_v4_1week"
|
||||
)
|
||||
|
||||
// localTimeSeriesTable maps a distributed time series table to its shard-local
|
||||
// table. Samples and time series shard on the same key
|
||||
// (cityHash64(env, temporality, metric_name, fingerprint)), so a query whose
|
||||
// top-level FROM is the distributed samples table can join or semi-join the
|
||||
// local time series table inside each shard: the shard rewrite runs the
|
||||
// subquery against the shard's own series rows, which are exactly the series
|
||||
// of the shard's samples. No broadcast, no initiator-side join.
|
||||
func localTimeSeriesTable(distributed string) string {
|
||||
switch distributed {
|
||||
case distributedTimeSeriesV46hrs:
|
||||
return localTimeSeriesV46hrs
|
||||
case distributedTimeSeriesV41day:
|
||||
return localTimeSeriesV41day
|
||||
case distributedTimeSeriesV41week:
|
||||
return localTimeSeriesV41week
|
||||
default:
|
||||
return localTimeSeriesV4
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
oneHourInMilliseconds = time.Hour.Milliseconds()
|
||||
sixHoursInMilliseconds = time.Hour.Milliseconds() * 6
|
||||
oneDayInMilliseconds = time.Hour.Milliseconds() * 24
|
||||
oneWeekInMilliseconds = time.Hour.Milliseconds() * 24 * 7
|
||||
)
|
||||
|
||||
// timeSeriesTableFor returns the adjusted start and the time series table for
|
||||
// the window. Time series tables hold one row per (fingerprint, bucket), with
|
||||
// bucket granularities of 1h, 6h, 1d and 1w; the start is rounded down to the
|
||||
// bucket boundary so a window beginning mid-bucket still matches the bucket's
|
||||
// row.
|
||||
func timeSeriesTableFor(start, end int64) (int64, string) {
|
||||
switch {
|
||||
case end-start < sixHoursInMilliseconds:
|
||||
return start - (start % oneHourInMilliseconds), distributedTimeSeriesV4
|
||||
case end-start < oneDayInMilliseconds:
|
||||
return start - (start % sixHoursInMilliseconds), distributedTimeSeriesV46hrs
|
||||
case end-start < oneWeekInMilliseconds:
|
||||
return start - (start % oneDayInMilliseconds), distributedTimeSeriesV41day
|
||||
default:
|
||||
return start - (start % oneWeekInMilliseconds), distributedTimeSeriesV41week
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,11 @@ type ActiveQueryTrackerConfig struct {
|
||||
MaxConcurrent int `mapstructure:"max_concurrent"`
|
||||
}
|
||||
|
||||
type ClickhouseV2Config struct {
|
||||
MaxFetchedSeries int `mapstructure:"max_fetched_series"`
|
||||
MaxFetchedSamples int64 `mapstructure:"max_fetched_samples"`
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ActiveQueryTrackerConfig ActiveQueryTrackerConfig `mapstructure:"active_query_tracker"`
|
||||
|
||||
@@ -24,6 +29,9 @@ type Config struct {
|
||||
|
||||
// Timeout is the maximum time a query is allowed to run before being aborted.
|
||||
Timeout time.Duration `mapstructure:"timeout"`
|
||||
|
||||
// ClickhouseV2 configures the clickhousev2 provider.
|
||||
ClickhouseV2 ClickhouseV2Config `mapstructure:"clickhousev2"`
|
||||
}
|
||||
|
||||
func NewConfigFactory() factory.ConfigFactory {
|
||||
@@ -38,6 +46,10 @@ func newConfig() factory.Config {
|
||||
MaxConcurrent: 20,
|
||||
},
|
||||
Timeout: 2 * time.Minute,
|
||||
ClickhouseV2: ClickhouseV2Config{
|
||||
MaxFetchedSeries: 500_000,
|
||||
MaxFetchedSamples: 50_000_000,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,6 +57,9 @@ func (c Config) Validate() error {
|
||||
if c.Timeout <= 0 {
|
||||
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "prometheus::timeout must be greater than 0")
|
||||
}
|
||||
if c.ClickhouseV2.MaxFetchedSeries < 0 || c.ClickhouseV2.MaxFetchedSamples < 0 {
|
||||
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "prometheus::clickhousev2 limits must not be negative")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
49
pkg/prometheus/traits.go
Normal file
49
pkg/prometheus/traits.go
Normal file
@@ -0,0 +1,49 @@
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/prometheus/prometheus/promql/parser"
|
||||
)
|
||||
|
||||
type queryTraitsKey struct{}
|
||||
|
||||
// QueryTraits carries per-query facts a storage implementation cannot derive
|
||||
// from SelectHints alone. Call sites that parse the PromQL expression attach
|
||||
// traits to the context before handing it to the engine; storages treat a
|
||||
// missing traits value as "unknown" and stay conservative.
|
||||
type QueryTraits struct {
|
||||
// SubqueryFree is true when the query contains no subquery expression.
|
||||
// Subquery selectors are evaluated at the subquery's own step, but
|
||||
// SelectHints.Step always carries the top-level step, so step-aligned
|
||||
// storage optimizations (e.g. keeping only the last sample per step
|
||||
// bucket) are safe only when this is true.
|
||||
SubqueryFree bool
|
||||
}
|
||||
|
||||
// DetectQueryTraits derives QueryTraits from a parsed PromQL expression.
|
||||
func DetectQueryTraits(expr parser.Expr) QueryTraits {
|
||||
subqueryFree := true
|
||||
parser.Inspect(expr, func(node parser.Node, _ []parser.Node) error {
|
||||
if _, ok := node.(*parser.SubqueryExpr); ok {
|
||||
subqueryFree = false
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return QueryTraits{SubqueryFree: subqueryFree}
|
||||
}
|
||||
|
||||
// NewContextWithQueryTraits returns a context carrying the given traits.
|
||||
func NewContextWithQueryTraits(ctx context.Context, traits QueryTraits) context.Context {
|
||||
return context.WithValue(ctx, queryTraitsKey{}, traits)
|
||||
}
|
||||
|
||||
// QueryTraitsFromContext returns the traits attached to ctx, if any.
|
||||
//
|
||||
// Context is used here, unlike for backend selection, because traits must
|
||||
// cross the promql engine to reach storage.Querier.Select, and the engine's
|
||||
// interfaces offer no other channel; the alternative is a Prometheus fork.
|
||||
func QueryTraitsFromContext(ctx context.Context) (QueryTraits, bool) {
|
||||
traits, ok := ctx.Value(queryTraitsKey{}).(QueryTraits)
|
||||
return traits, ok
|
||||
}
|
||||
@@ -22,13 +22,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/variables"
|
||||
)
|
||||
|
||||
type Handler interface {
|
||||
QueryRange(rw http.ResponseWriter, req *http.Request)
|
||||
QueryRangePreview(rw http.ResponseWriter, req *http.Request)
|
||||
QueryRawStream(rw http.ResponseWriter, req *http.Request)
|
||||
ReplaceVariables(rw http.ResponseWriter, req *http.Request)
|
||||
}
|
||||
|
||||
type handler struct {
|
||||
set factory.ProviderSettings
|
||||
analytics analytics.Analytics
|
||||
|
||||
@@ -15,14 +15,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// BucketCache is the interface for bucket-based caching.
|
||||
type BucketCache interface {
|
||||
// cached portion + list of gaps to fetch
|
||||
GetMissRanges(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step) (cached *qbtypes.Result, missing []*qbtypes.TimeRange)
|
||||
// store fresh buckets for future hits
|
||||
Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result)
|
||||
}
|
||||
|
||||
// bucketCache implements the BucketCache interface.
|
||||
type bucketCache struct {
|
||||
cache cache.Cache
|
||||
|
||||
@@ -11,8 +11,8 @@ import (
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -277,12 +277,12 @@ func (q *builderQuery[T]) narrowWindowByTraceID(ctx context.Context, fromMS, toM
|
||||
return fromMS, toMS, true, ""
|
||||
}
|
||||
|
||||
traceIDs, found := tracestelemetryschema.ExtractTraceIDsFromFilter(q.spec.Filter.Expression)
|
||||
traceIDs, found := telemetrytraces.ExtractTraceIDsFromFilter(q.spec.Filter.Expression)
|
||||
if !found || len(traceIDs) == 0 {
|
||||
return fromMS, toMS, true, ""
|
||||
}
|
||||
|
||||
finder := tracestelemetryschema.NewTraceTimeRangeFinder(q.telemetryStore)
|
||||
finder := telemetrytraces.NewTraceTimeRangeFinder(q.telemetryStore)
|
||||
traceStart, traceEnd, exists, err := finder.GetTraceTimeRangeMulti(ctx, traceIDs)
|
||||
if err != nil {
|
||||
return fromMS, toMS, true, ""
|
||||
|
||||
@@ -6,18 +6,18 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
func (q *querier) Collect(ctx context.Context, _ valuer.UUID) (map[string]any, error) {
|
||||
stats := make(map[string]any)
|
||||
|
||||
tracesTable := fmt.Sprintf("%s.%s", tracestelemetryschema.DBName, tracestelemetryschema.SpanIndexV3TableName)
|
||||
logsTable := fmt.Sprintf("%s.%s", logstelemetryschema.DBName, logstelemetryschema.LogsV2TableName)
|
||||
metricsTable := fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4TableName)
|
||||
tracesTable := fmt.Sprintf("%s.%s", telemetrytraces.DBName, telemetrytraces.SpanIndexV3TableName)
|
||||
logsTable := fmt.Sprintf("%s.%s", telemetrylogs.DBName, telemetrylogs.LogsV2TableName)
|
||||
metricsTable := fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4TableName)
|
||||
|
||||
var (
|
||||
traces uint64
|
||||
|
||||
@@ -9,6 +9,12 @@ import (
|
||||
|
||||
const DefaultMaxConcurrentQueries = 8
|
||||
|
||||
type SkipResourceFingerprint struct {
|
||||
Enabled bool `yaml:"enabled" mapstructure:"enabled"`
|
||||
// If count of fingerprint is above threshold, skip the fingerprint subquery and filter on main table instead.
|
||||
Threshold uint64 `yaml:"threshold" mapstructure:"threshold"`
|
||||
}
|
||||
|
||||
// Config represents the configuration for the querier.
|
||||
type Config struct {
|
||||
// CacheTTL is the TTL for cached query results
|
||||
@@ -17,6 +23,8 @@ type Config struct {
|
||||
FluxInterval time.Duration `yaml:"flux_interval" mapstructure:"flux_interval"`
|
||||
// MaxConcurrentQueries is the maximum number of concurrent queries for missing ranges
|
||||
MaxConcurrentQueries int `yaml:"max_concurrent_queries" mapstructure:"max_concurrent_queries"`
|
||||
// SkipResourceFingerprint configures when the resource fingerprint subquery is skipped in favor of main-table filtering.
|
||||
SkipResourceFingerprint SkipResourceFingerprint `yaml:"skip_resource_fingerprint" mapstructure:"skip_resource_fingerprint"`
|
||||
// LogTraceIDWindowPadding is the padding added to narrowed down timerange from trace summary to logs with trace_id filter.
|
||||
LogTraceIDWindowPadding time.Duration `yaml:"log_trace_id_window_padding" mapstructure:"log_trace_id_window_padding"`
|
||||
}
|
||||
@@ -31,7 +39,11 @@ func newConfig() factory.Config {
|
||||
// Default values
|
||||
CacheTTL: 168 * time.Hour,
|
||||
FluxInterval: 5 * time.Minute,
|
||||
MaxConcurrentQueries: DefaultMaxConcurrentQueries,
|
||||
MaxConcurrentQueries: DefaultMaxConcurrentQueries,
|
||||
SkipResourceFingerprint: SkipResourceFingerprint{
|
||||
Enabled: false,
|
||||
Threshold: 100000,
|
||||
},
|
||||
LogTraceIDWindowPadding: 5 * time.Minute,
|
||||
}
|
||||
}
|
||||
@@ -47,6 +59,9 @@ func (c Config) Validate() error {
|
||||
if c.MaxConcurrentQueries <= 0 {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "max_concurrent_queries must be positive, got %v", c.MaxConcurrentQueries)
|
||||
}
|
||||
if c.SkipResourceFingerprint.Enabled && c.SkipResourceFingerprint.Threshold == 0 {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "skip_resource_fingerprint.threshold must be > 0 when enabled")
|
||||
}
|
||||
if c.LogTraceIDWindowPadding < 0 {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "log_trace_id_window_padding must not be negative, got %v", c.LogTraceIDWindowPadding)
|
||||
}
|
||||
|
||||
35
pkg/querier/interfaces.go
Normal file
35
pkg/querier/interfaces.go
Normal file
@@ -0,0 +1,35 @@
|
||||
package querier
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/statsreporter"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// Querier interface defines the contract for querying data.
|
||||
type Querier interface {
|
||||
QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (*qbtypes.QueryRangeResponse, error)
|
||||
QueryRawStream(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, client *qbtypes.RawStream)
|
||||
statsreporter.StatsCollector
|
||||
// QueryRangePreview validates and renders the queries without executing them.
|
||||
QueryRangePreview(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, opts qbtypes.QueryRangePreviewOptions) (*qbtypes.QueryRangePreviewResponse, error)
|
||||
}
|
||||
|
||||
// BucketCache is the interface for bucket-based caching.
|
||||
type BucketCache interface {
|
||||
// cached portion + list of gaps to fetch
|
||||
GetMissRanges(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step) (cached *qbtypes.Result, missing []*qbtypes.TimeRange)
|
||||
// store fresh buckets for future hits
|
||||
Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result)
|
||||
}
|
||||
|
||||
type Handler interface {
|
||||
QueryRange(rw http.ResponseWriter, req *http.Request)
|
||||
// QueryRangePreview is the dry-run endpoint: validate and render without executing.
|
||||
QueryRangePreview(rw http.ResponseWriter, req *http.Request)
|
||||
QueryRawStream(rw http.ResponseWriter, req *http.Request)
|
||||
ReplaceVariables(rw http.ResponseWriter, req *http.Request)
|
||||
}
|
||||
@@ -331,11 +331,14 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
|
||||
return nil, errors.WrapInternalf(promErr, errors.CodeInternal, "error getting matrix from promql query %q", query)
|
||||
}
|
||||
|
||||
// Hide only known SigNoz storage keys: label names are user data and may
|
||||
// legitimately start with "__" (e.g. __address__), so a blanket dunder
|
||||
// strip mangles user labelsets. The __scope./__resource. prefixes cover
|
||||
// every exporter version's keys.
|
||||
excludeLabel := func(labelName string) bool {
|
||||
if labelName == "__name__" {
|
||||
return false
|
||||
}
|
||||
return strings.HasPrefix(labelName, "__") || labelName == "fingerprint"
|
||||
return labelName == "__temporality__" ||
|
||||
strings.HasPrefix(labelName, "__scope.") ||
|
||||
strings.HasPrefix(labelName, "__resource.")
|
||||
}
|
||||
|
||||
var series []*qbv5.TimeSeries
|
||||
|
||||
@@ -21,8 +21,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statsreporter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
@@ -37,15 +35,6 @@ var (
|
||||
intervalWarn = "Query %s is requesting aggregation interval %v seconds, which is smaller than the minimum allowed interval of %v seconds for selected time range. Using the minimum instead"
|
||||
)
|
||||
|
||||
// Querier interface defines the contract for querying data.
|
||||
type Querier interface {
|
||||
QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (*qbtypes.QueryRangeResponse, error)
|
||||
QueryRawStream(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, client *qbtypes.RawStream)
|
||||
statsreporter.StatsCollector
|
||||
// QueryRangePreview validates and renders the queries without executing them.
|
||||
QueryRangePreview(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, opts qbtypes.QueryRangePreviewOptions) (*qbtypes.QueryRangePreviewResponse, error)
|
||||
}
|
||||
|
||||
type querier struct {
|
||||
logger *slog.Logger
|
||||
fl flagger.Flagger
|
||||
@@ -71,7 +60,12 @@ func New(
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
promEngine prometheus.Prometheus,
|
||||
builders *statementbuilder.Builders,
|
||||
traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
|
||||
logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
|
||||
auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
|
||||
metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation],
|
||||
meterStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation],
|
||||
traceOperatorStmtBuilder qbtypes.TraceOperatorStatementBuilder,
|
||||
bucketCache BucketCache,
|
||||
flagger flagger.Flagger,
|
||||
logTraceIDWindowPadding time.Duration,
|
||||
@@ -81,21 +75,18 @@ func New(
|
||||
if maxConcurrentQueries <= 0 {
|
||||
maxConcurrentQueries = DefaultMaxConcurrentQueries
|
||||
}
|
||||
if builders == nil {
|
||||
builders = &statementbuilder.Builders{}
|
||||
}
|
||||
return &querier{
|
||||
logger: querierSettings.Logger(),
|
||||
fl: flagger,
|
||||
telemetryStore: telemetryStore,
|
||||
metadataStore: metadataStore,
|
||||
promEngine: promEngine,
|
||||
traceStmtBuilder: builders.Trace,
|
||||
logStmtBuilder: builders.Log,
|
||||
auditStmtBuilder: builders.Audit,
|
||||
metricStmtBuilder: builders.Metric,
|
||||
meterStmtBuilder: builders.Meter,
|
||||
traceOperatorStmtBuilder: builders.TraceOperator,
|
||||
traceStmtBuilder: traceStmtBuilder,
|
||||
logStmtBuilder: logStmtBuilder,
|
||||
auditStmtBuilder: auditStmtBuilder,
|
||||
metricStmtBuilder: metricStmtBuilder,
|
||||
meterStmtBuilder: meterStmtBuilder,
|
||||
traceOperatorStmtBuilder: traceOperatorStmtBuilder,
|
||||
bucketCache: bucketCache,
|
||||
liveDataRefresh: 5 * time.Second,
|
||||
builderConfig: builderConfig{
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
@@ -49,7 +48,12 @@ func TestQueryRange_MetricTypeMissing(t *testing.T) {
|
||||
nil, // telemetryStore
|
||||
metadataStore,
|
||||
nil, // prometheus
|
||||
nil, // builders
|
||||
nil, // traceStmtBuilder
|
||||
nil, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
nil, // metricStmtBuilder
|
||||
nil, // meterStmtBuilder
|
||||
nil, // traceOperatorStmtBuilder
|
||||
nil, // bucketCache
|
||||
flaggertest.New(t), // flagger
|
||||
0, // logTraceIDWindowPadding
|
||||
@@ -115,12 +119,17 @@ func TestQueryRange_MetricTypeFromStore(t *testing.T) {
|
||||
providerSettings,
|
||||
telemetryStore,
|
||||
metadataStore,
|
||||
nil, // prometheus
|
||||
&statementbuilder.Builders{Metric: &mockMetricStmtBuilder{}},
|
||||
nil, // bucketCache
|
||||
flaggertest.New(t), // flagger
|
||||
0, // logTraceIDWindowPadding
|
||||
0, // maxConcurrentQueries
|
||||
nil, // prometheus
|
||||
nil, // traceStmtBuilder
|
||||
nil, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
&mockMetricStmtBuilder{}, // metricStmtBuilder
|
||||
nil, // meterStmtBuilder
|
||||
nil, // traceOperatorStmtBuilder
|
||||
nil, // bucketCache
|
||||
flaggertest.New(t), // flagger
|
||||
0, // logTraceIDWindowPadding
|
||||
0, // maxConcurrentQueries
|
||||
)
|
||||
|
||||
req := &qbtypes.QueryRangeRequest{
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package queriertest
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// MockQuerier implements querier.Querier for testing. Each behavior can be
|
||||
// overridden via a function field; unset fields return empty, non-error values.
|
||||
type MockQuerier struct {
|
||||
QueryRangeFunc func(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (*qbtypes.QueryRangeResponse, error)
|
||||
QueryRawStreamFunc func(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, client *qbtypes.RawStream)
|
||||
QueryRangePreviewFunc func(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, opts qbtypes.QueryRangePreviewOptions) (*qbtypes.QueryRangePreviewResponse, error)
|
||||
CollectFunc func(ctx context.Context, orgID valuer.UUID) (map[string]any, error)
|
||||
}
|
||||
|
||||
var _ querier.Querier = (*MockQuerier)(nil)
|
||||
|
||||
// NewMockQuerier creates a new MockQuerier.
|
||||
func NewMockQuerier() *MockQuerier {
|
||||
return &MockQuerier{}
|
||||
}
|
||||
|
||||
func (m *MockQuerier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (*qbtypes.QueryRangeResponse, error) {
|
||||
if m.QueryRangeFunc != nil {
|
||||
return m.QueryRangeFunc(ctx, orgID, req)
|
||||
}
|
||||
return &qbtypes.QueryRangeResponse{}, nil
|
||||
}
|
||||
|
||||
func (m *MockQuerier) QueryRawStream(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, client *qbtypes.RawStream) {
|
||||
if m.QueryRawStreamFunc != nil {
|
||||
m.QueryRawStreamFunc(ctx, orgID, req, client)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *MockQuerier) QueryRangePreview(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, opts qbtypes.QueryRangePreviewOptions) (*qbtypes.QueryRangePreviewResponse, error) {
|
||||
if m.QueryRangePreviewFunc != nil {
|
||||
return m.QueryRangePreviewFunc(ctx, orgID, req, opts)
|
||||
}
|
||||
return &qbtypes.QueryRangePreviewResponse{}, nil
|
||||
}
|
||||
|
||||
func (m *MockQuerier) Collect(ctx context.Context, orgID valuer.UUID) (map[string]any, error) {
|
||||
if m.CollectFunc != nil {
|
||||
return m.CollectFunc(ctx, orgID)
|
||||
}
|
||||
return map[string]any{}, nil
|
||||
}
|
||||
@@ -3,44 +3,192 @@ package signozquerier
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/cache"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryaudit"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetadata"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
)
|
||||
|
||||
// NewFactory creates a new factory for the signoz querier provider. The query
|
||||
// stack (metadata store, statement builders, bucket cache) is assembled once in
|
||||
// signoz.go and injected here.
|
||||
// NewFactory creates a new factory for the signoz querier provider.
|
||||
func NewFactory(
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
prometheus prometheus.Prometheus,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
builders *statementbuilder.Builders,
|
||||
bucketCache querier.BucketCache,
|
||||
cache cache.Cache,
|
||||
flagger flagger.Flagger,
|
||||
) factory.ProviderFactory[querier.Querier, querier.Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("signoz"),
|
||||
func(
|
||||
_ context.Context,
|
||||
ctx context.Context,
|
||||
settings factory.ProviderSettings,
|
||||
cfg querier.Config,
|
||||
) (querier.Querier, error) {
|
||||
return querier.New(
|
||||
settings,
|
||||
telemetryStore,
|
||||
metadataStore,
|
||||
prometheus,
|
||||
builders,
|
||||
bucketCache,
|
||||
flagger,
|
||||
cfg.LogTraceIDWindowPadding,
|
||||
cfg.MaxConcurrentQueries,
|
||||
), nil
|
||||
return newProvider(ctx, settings, cfg, telemetryStore, prometheus, cache, flagger)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func newProvider(
|
||||
_ context.Context,
|
||||
settings factory.ProviderSettings,
|
||||
cfg querier.Config,
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
prometheus prometheus.Prometheus,
|
||||
cache cache.Cache,
|
||||
flagger flagger.Flagger,
|
||||
) (querier.Querier, error) {
|
||||
|
||||
// Create telemetry metadata store
|
||||
telemetryMetadataStore := telemetrymetadata.NewTelemetryMetaStore(
|
||||
settings,
|
||||
telemetryStore,
|
||||
telemetrytraces.DBName,
|
||||
telemetrytraces.TagAttributesV2TableName,
|
||||
telemetrytraces.SpanAttributesKeysTblName,
|
||||
telemetrytraces.SpanIndexV3TableName,
|
||||
telemetrymetrics.DBName,
|
||||
telemetrymetrics.AttributesMetadataTableName,
|
||||
telemetrymeter.DBName,
|
||||
telemetrymeter.SamplesAgg1dTableName,
|
||||
telemetrylogs.DBName,
|
||||
telemetrylogs.LogsV2TableName,
|
||||
telemetrylogs.TagAttributesV2TableName,
|
||||
telemetrylogs.LogAttributeKeysTblName,
|
||||
telemetrylogs.LogResourceKeysTblName,
|
||||
telemetryaudit.DBName,
|
||||
telemetryaudit.AuditLogsTableName,
|
||||
telemetryaudit.TagAttributesTableName,
|
||||
telemetryaudit.LogAttributeKeysTblName,
|
||||
telemetryaudit.LogResourceKeysTblName,
|
||||
telemetrymetadata.DBName,
|
||||
telemetrymetadata.AttributesMetadataTableName,
|
||||
telemetrymetadata.ColumnEvolutionMetadataTableName,
|
||||
flagger,
|
||||
)
|
||||
|
||||
// Create trace statement builder
|
||||
traceFieldMapper := telemetrytraces.NewFieldMapper()
|
||||
traceConditionBuilder := telemetrytraces.NewConditionBuilder(traceFieldMapper)
|
||||
|
||||
traceAggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, traceFieldMapper, traceConditionBuilder, flagger)
|
||||
traceStmtBuilder := telemetrytraces.NewTraceQueryStatementBuilder(
|
||||
settings,
|
||||
telemetryMetadataStore,
|
||||
traceFieldMapper,
|
||||
traceConditionBuilder,
|
||||
traceAggExprRewriter,
|
||||
telemetryStore,
|
||||
flagger,
|
||||
cfg.SkipResourceFingerprint.Enabled,
|
||||
cfg.SkipResourceFingerprint.Threshold,
|
||||
)
|
||||
|
||||
// Create trace operator statement builder
|
||||
traceOperatorStmtBuilder := telemetrytraces.NewTraceOperatorStatementBuilder(
|
||||
settings,
|
||||
telemetryMetadataStore,
|
||||
traceFieldMapper,
|
||||
traceConditionBuilder,
|
||||
traceStmtBuilder,
|
||||
traceAggExprRewriter,
|
||||
flagger,
|
||||
)
|
||||
|
||||
// Create log statement builder
|
||||
logFieldMapper := telemetrylogs.NewFieldMapper(flagger)
|
||||
logConditionBuilder := telemetrylogs.NewConditionBuilder(logFieldMapper, flagger)
|
||||
logAggExprRewriter := querybuilder.NewAggExprRewriter(
|
||||
settings,
|
||||
telemetrylogs.DefaultFullTextColumn,
|
||||
logFieldMapper,
|
||||
logConditionBuilder,
|
||||
flagger,
|
||||
)
|
||||
logStmtBuilder := telemetrylogs.NewLogQueryStatementBuilder(
|
||||
settings,
|
||||
telemetryMetadataStore,
|
||||
logFieldMapper,
|
||||
logConditionBuilder,
|
||||
logAggExprRewriter,
|
||||
telemetrylogs.DefaultFullTextColumn,
|
||||
flagger,
|
||||
telemetryStore,
|
||||
cfg.SkipResourceFingerprint.Enabled,
|
||||
cfg.SkipResourceFingerprint.Threshold,
|
||||
)
|
||||
|
||||
// Create audit statement builder
|
||||
auditFieldMapper := telemetryaudit.NewFieldMapper()
|
||||
auditConditionBuilder := telemetryaudit.NewConditionBuilder(auditFieldMapper)
|
||||
auditAggExprRewriter := querybuilder.NewAggExprRewriter(
|
||||
settings,
|
||||
telemetryaudit.DefaultFullTextColumn,
|
||||
auditFieldMapper,
|
||||
auditConditionBuilder,
|
||||
flagger,
|
||||
)
|
||||
auditStmtBuilder := telemetryaudit.NewAuditQueryStatementBuilder(
|
||||
settings,
|
||||
telemetryMetadataStore,
|
||||
auditFieldMapper,
|
||||
auditConditionBuilder,
|
||||
auditAggExprRewriter,
|
||||
telemetryaudit.DefaultFullTextColumn,
|
||||
flagger,
|
||||
)
|
||||
|
||||
// Create metric statement builder
|
||||
metricFieldMapper := telemetrymetrics.NewFieldMapper()
|
||||
metricConditionBuilder := telemetrymetrics.NewConditionBuilder(metricFieldMapper)
|
||||
metricStmtBuilder := telemetrymetrics.NewMetricQueryStatementBuilder(
|
||||
settings,
|
||||
telemetryMetadataStore,
|
||||
metricFieldMapper,
|
||||
metricConditionBuilder,
|
||||
flagger,
|
||||
)
|
||||
|
||||
// Create meter statement builder
|
||||
meterStmtBuilder := telemetrymeter.NewMeterQueryStatementBuilder(
|
||||
settings,
|
||||
telemetryMetadataStore,
|
||||
metricFieldMapper,
|
||||
metricConditionBuilder,
|
||||
metricStmtBuilder,
|
||||
)
|
||||
|
||||
// Create bucket cache
|
||||
bucketCache := querier.NewBucketCache(
|
||||
settings,
|
||||
cache,
|
||||
cfg.CacheTTL,
|
||||
cfg.FluxInterval,
|
||||
)
|
||||
|
||||
// Create and return the querier
|
||||
return querier.New(
|
||||
settings,
|
||||
telemetryStore,
|
||||
telemetryMetadataStore,
|
||||
prometheus,
|
||||
traceStmtBuilder,
|
||||
logStmtBuilder,
|
||||
auditStmtBuilder,
|
||||
metricStmtBuilder,
|
||||
meterStmtBuilder,
|
||||
traceOperatorStmtBuilder,
|
||||
bucketCache,
|
||||
flagger,
|
||||
cfg.LogTraceIDWindowPadding,
|
||||
cfg.MaxConcurrentQueries,
|
||||
), nil
|
||||
}
|
||||
|
||||
@@ -18,13 +18,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/querier/signozquerier"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore/sqlstoretest"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/auditstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/logsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/meterstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/tracesstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetadata"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -112,31 +105,7 @@ func NewTestManager(t *testing.T, testOpts *TestManagerOptions) *Manager {
|
||||
}
|
||||
|
||||
// Create querier with test values
|
||||
metadataStore := telemetrymetadata.NewTelemetryMetaStore(providerSettings, telemetryStore, flagger)
|
||||
cfg := statementbuilder.Config{}
|
||||
ctx := context.Background()
|
||||
traceStmtBuilder, err := tracesstatementbuilder.NewFactory(telemetryStore, metadataStore, flagger).New(ctx, providerSettings, cfg)
|
||||
require.NoError(t, err)
|
||||
traceOperatorStmtBuilder, err := tracesstatementbuilder.NewOperatorFactory(telemetryStore, metadataStore, flagger).New(ctx, providerSettings, cfg)
|
||||
require.NoError(t, err)
|
||||
logStmtBuilder, err := logsstatementbuilder.NewFactory(telemetryStore, metadataStore, flagger).New(ctx, providerSettings, cfg)
|
||||
require.NoError(t, err)
|
||||
auditStmtBuilder, err := auditstatementbuilder.NewFactory(metadataStore, flagger).New(ctx, providerSettings, cfg)
|
||||
require.NoError(t, err)
|
||||
metricStmtBuilder, err := metricsstatementbuilder.NewFactory(metadataStore, flagger).New(ctx, providerSettings, cfg)
|
||||
require.NoError(t, err)
|
||||
meterStmtBuilder, err := meterstatementbuilder.NewFactory(metadataStore, flagger).New(ctx, providerSettings, cfg)
|
||||
require.NoError(t, err)
|
||||
builders := &statementbuilder.Builders{
|
||||
Trace: traceStmtBuilder,
|
||||
TraceOperator: traceOperatorStmtBuilder,
|
||||
Log: logStmtBuilder,
|
||||
Audit: auditStmtBuilder,
|
||||
Metric: metricStmtBuilder,
|
||||
Meter: meterStmtBuilder,
|
||||
}
|
||||
bucketCache := querier.NewBucketCache(providerSettings, cache, 0, 0)
|
||||
providerFactory := signozquerier.NewFactory(telemetryStore, prometheus, metadataStore, builders, bucketCache, flagger)
|
||||
providerFactory := signozquerier.NewFactory(telemetryStore, prometheus, cache, flagger)
|
||||
mockQuerier, err := providerFactory.New(context.Background(), providerSettings, querier.Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
|
||||
@@ -8,11 +8,11 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/logsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/tracesstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -24,7 +24,7 @@ func prepareQuerierForMetrics(t *testing.T, telemetryStore telemetrystore.Teleme
|
||||
providerSettings := instrumentationtest.New().ToProviderSettings()
|
||||
metadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
|
||||
fl, err := flagger.New(
|
||||
flagger, err := flagger.New(
|
||||
context.Background(),
|
||||
instrumentationtest.New().ToProviderSettings(),
|
||||
flagger.Config{},
|
||||
@@ -32,17 +32,29 @@ func prepareQuerierForMetrics(t *testing.T, telemetryStore telemetrystore.Teleme
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
metricStmtBuilder, err := metricsstatementbuilder.NewFactory(metadataStore, fl).New(context.Background(), providerSettings, statementbuilder.Config{})
|
||||
require.NoError(t, err)
|
||||
metricFieldMapper := telemetrymetrics.NewFieldMapper()
|
||||
metricConditionBuilder := telemetrymetrics.NewConditionBuilder(metricFieldMapper)
|
||||
metricStmtBuilder := telemetrymetrics.NewMetricQueryStatementBuilder(
|
||||
providerSettings,
|
||||
metadataStore,
|
||||
metricFieldMapper,
|
||||
metricConditionBuilder,
|
||||
flagger,
|
||||
)
|
||||
|
||||
return querier.New(
|
||||
providerSettings,
|
||||
telemetryStore,
|
||||
metadataStore,
|
||||
nil, // prometheus
|
||||
&statementbuilder.Builders{Metric: metricStmtBuilder},
|
||||
nil, // traceStmtBuilder
|
||||
nil, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
metricStmtBuilder,
|
||||
nil, // meterStmtBuilder
|
||||
nil, // traceOperatorStmtBuilder
|
||||
nil, // bucketCache
|
||||
fl,
|
||||
flagger,
|
||||
0,
|
||||
0, // maxConcurrentQueries (0 means default)
|
||||
), metadataStore
|
||||
@@ -61,16 +73,40 @@ func prepareQuerierForLogs(t *testing.T, telemetryStore telemetrystore.Telemetry
|
||||
metadataStore.KeysMap = keysMap
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
logStmtBuilder, err := logsstatementbuilder.NewFactory(nil, metadataStore, fl).New(context.Background(), providerSettings, statementbuilder.Config{})
|
||||
require.NoError(t, err)
|
||||
logFieldMapper := telemetrylogs.NewFieldMapper(fl)
|
||||
logConditionBuilder := telemetrylogs.NewConditionBuilder(logFieldMapper, fl)
|
||||
logAggExprRewriter := querybuilder.NewAggExprRewriter(
|
||||
providerSettings,
|
||||
telemetrylogs.DefaultFullTextColumn,
|
||||
logFieldMapper,
|
||||
logConditionBuilder,
|
||||
fl,
|
||||
)
|
||||
logStmtBuilder := telemetrylogs.NewLogQueryStatementBuilder(
|
||||
providerSettings,
|
||||
metadataStore,
|
||||
logFieldMapper,
|
||||
logConditionBuilder,
|
||||
logAggExprRewriter,
|
||||
telemetrylogs.DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
100000,
|
||||
)
|
||||
|
||||
return querier.New(
|
||||
providerSettings,
|
||||
telemetryStore,
|
||||
metadataStore,
|
||||
nil, // prometheus
|
||||
&statementbuilder.Builders{Log: logStmtBuilder},
|
||||
nil, // bucketCache
|
||||
nil, // prometheus
|
||||
nil, // traceStmtBuilder
|
||||
logStmtBuilder, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
nil, // metricStmtBuilder
|
||||
nil, // meterStmtBuilder
|
||||
nil, // traceOperatorStmtBuilder
|
||||
nil, // bucketCache
|
||||
fl,
|
||||
5*time.Minute, // logTraceIDWindowPadding
|
||||
0, // maxConcurrentQueries (0 means default)
|
||||
@@ -90,17 +126,36 @@ func prepareQuerierForTraces(t *testing.T, telemetryStore telemetrystore.Telemet
|
||||
}
|
||||
metadataStore.KeysMap = keysMap
|
||||
|
||||
// Create trace statement builder
|
||||
traceFieldMapper := telemetrytraces.NewFieldMapper()
|
||||
traceConditionBuilder := telemetrytraces.NewConditionBuilder(traceFieldMapper)
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
traceStmtBuilder, err := tracesstatementbuilder.NewFactory(telemetryStore, metadataStore, fl).New(context.Background(), providerSettings, statementbuilder.Config{})
|
||||
require.NoError(t, err)
|
||||
traceAggExprRewriter := querybuilder.NewAggExprRewriter(providerSettings, nil, traceFieldMapper, traceConditionBuilder, fl)
|
||||
traceStmtBuilder := telemetrytraces.NewTraceQueryStatementBuilder(
|
||||
providerSettings,
|
||||
metadataStore,
|
||||
traceFieldMapper,
|
||||
traceConditionBuilder,
|
||||
traceAggExprRewriter,
|
||||
telemetryStore,
|
||||
fl,
|
||||
false,
|
||||
100000,
|
||||
)
|
||||
|
||||
return querier.New(
|
||||
providerSettings,
|
||||
telemetryStore,
|
||||
metadataStore,
|
||||
nil, // prometheus
|
||||
&statementbuilder.Builders{Trace: traceStmtBuilder},
|
||||
nil, // bucketCache
|
||||
nil, // prometheus
|
||||
traceStmtBuilder, // traceStmtBuilder
|
||||
nil, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
nil, // metricStmtBuilder
|
||||
nil, // meterStmtBuilder
|
||||
nil, // traceOperatorStmtBuilder
|
||||
nil, // bucketCache
|
||||
fl,
|
||||
0,
|
||||
0, // maxConcurrentQueries (0 means default)
|
||||
|
||||
@@ -39,7 +39,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/sqlmigrator"
|
||||
"github.com/SigNoz/signoz/pkg/sqlschema"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statsreporter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/tokenizer"
|
||||
@@ -98,9 +97,6 @@ type Config struct {
|
||||
// Querier config
|
||||
Querier querier.Config `mapstructure:"querier"`
|
||||
|
||||
// StatementBuilder config
|
||||
StatementBuilder statementbuilder.Config `mapstructure:"statementbuilder"`
|
||||
|
||||
// Ruler config
|
||||
Ruler ruler.Config `mapstructure:"ruler"`
|
||||
|
||||
@@ -170,7 +166,6 @@ func NewConfig(ctx context.Context, logger *slog.Logger, resolverConfig config.R
|
||||
prometheus.NewConfigFactory(),
|
||||
alertmanager.NewConfigFactory(),
|
||||
querier.NewConfigFactory(),
|
||||
statementbuilder.NewConfigFactory(),
|
||||
ruler.NewConfigFactory(),
|
||||
emailing.NewConfigFactory(),
|
||||
sharder.NewConfigFactory(),
|
||||
|
||||
@@ -56,7 +56,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore/sqlitesqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore/sqlstorehook"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statsreporter"
|
||||
"github.com/SigNoz/signoz/pkg/statsreporter/analyticsstatsreporter"
|
||||
"github.com/SigNoz/signoz/pkg/statsreporter/noopstatsreporter"
|
||||
@@ -70,7 +69,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/version"
|
||||
"github.com/SigNoz/signoz/pkg/web"
|
||||
"github.com/SigNoz/signoz/pkg/web/noopweb"
|
||||
@@ -287,9 +285,9 @@ func NewStatsReporterProviderFactories(aggregator statsreporter.Aggregator, orgG
|
||||
)
|
||||
}
|
||||
|
||||
func NewQuerierProviderFactories(telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, metadataStore telemetrytypes.MetadataStore, builders *statementbuilder.Builders, bucketCache querier.BucketCache, flagger flagger.Flagger) factory.NamedMap[factory.ProviderFactory[querier.Querier, querier.Config]] {
|
||||
func NewQuerierProviderFactories(telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, cache cache.Cache, flagger flagger.Flagger) factory.NamedMap[factory.ProviderFactory[querier.Querier, querier.Config]] {
|
||||
return factory.MustNewNamedMap(
|
||||
signozquerier.NewFactory(telemetryStore, prometheus, metadataStore, builders, bucketCache, flagger),
|
||||
signozquerier.NewFactory(telemetryStore, prometheus, cache, flagger),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -48,15 +48,14 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/sqlmigrator"
|
||||
"github.com/SigNoz/signoz/pkg/sqlschema"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/auditstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/logsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/meterstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/tracesstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statsreporter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryaudit"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetadata"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
pkgtokenizer "github.com/SigNoz/signoz/pkg/tokenizer"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -97,61 +96,6 @@ type SigNoz struct {
|
||||
MeterReporter meterreporter.Reporter
|
||||
}
|
||||
|
||||
// newQueryStack assembles the query stack once: the shared telemetry metadata
|
||||
// store, the per-signal statement builders, and the bucket cache. The metadata
|
||||
// store is returned so callers can reuse the single instance downstream.
|
||||
func newQueryStack(
|
||||
ctx context.Context,
|
||||
settings factory.ProviderSettings,
|
||||
config Config,
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
cache cache.Cache,
|
||||
fl flagger.Flagger,
|
||||
) (telemetrytypes.MetadataStore, *statementbuilder.Builders, querier.BucketCache, error) {
|
||||
metadataStore := telemetrymetadata.NewTelemetryMetaStore(settings, telemetryStore, fl)
|
||||
|
||||
// Composition root: build each per-signal statement-builder factory and assemble
|
||||
// the Builders bundle. This is the only place that imports the concrete sub-packages.
|
||||
cfg := config.StatementBuilder
|
||||
traceStmtBuilder, err := tracesstatementbuilder.NewFactory(telemetryStore, metadataStore, fl).New(ctx, settings, cfg)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
traceOperatorStmtBuilder, err := tracesstatementbuilder.NewOperatorFactory(telemetryStore, metadataStore, fl).New(ctx, settings, cfg)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
logStmtBuilder, err := logsstatementbuilder.NewFactory(telemetryStore, metadataStore, fl).New(ctx, settings, cfg)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
auditStmtBuilder, err := auditstatementbuilder.NewFactory(metadataStore, fl).New(ctx, settings, cfg)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
metricStmtBuilder, err := metricsstatementbuilder.NewFactory(metadataStore, fl).New(ctx, settings, cfg)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
meterStmtBuilder, err := meterstatementbuilder.NewFactory(metadataStore, fl).New(ctx, settings, cfg)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
builders := &statementbuilder.Builders{
|
||||
Trace: traceStmtBuilder,
|
||||
TraceOperator: traceOperatorStmtBuilder,
|
||||
Log: logStmtBuilder,
|
||||
Audit: auditStmtBuilder,
|
||||
Metric: metricStmtBuilder,
|
||||
Meter: meterStmtBuilder,
|
||||
}
|
||||
|
||||
bucketCache := querier.NewBucketCache(settings, cache, config.Querier.CacheTTL, config.Querier.FluxInterval)
|
||||
|
||||
return metadataStore, builders, bucketCache, nil
|
||||
}
|
||||
|
||||
func New(
|
||||
ctx context.Context,
|
||||
config Config,
|
||||
@@ -310,19 +254,12 @@ func New(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Assemble the query stack (metadata store, statement builders, bucket cache) once,
|
||||
// and reuse the single metadata store everywhere downstream.
|
||||
telemetryMetadataStore, builders, bucketCache, err := newQueryStack(ctx, providerSettings, config, telemetrystore, cache, flagger)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Initialize querier from the available querier provider factories
|
||||
querier, err := factory.NewProviderFromNamedMap(
|
||||
ctx,
|
||||
providerSettings,
|
||||
config.Querier,
|
||||
NewQuerierProviderFactories(telemetrystore, prometheus, telemetryMetadataStore, builders, bucketCache, flagger),
|
||||
NewQuerierProviderFactories(telemetrystore, prometheus, cache, flagger),
|
||||
config.Querier.Provider(),
|
||||
)
|
||||
if err != nil {
|
||||
@@ -490,6 +427,35 @@ func New(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Initialize telemetry metadata store
|
||||
// TODO: consolidate other telemetrymetadata.NewTelemetryMetaStore initializations to reuse this instance instead.
|
||||
telemetryMetadataStore := telemetrymetadata.NewTelemetryMetaStore(
|
||||
providerSettings,
|
||||
telemetrystore,
|
||||
telemetrytraces.DBName,
|
||||
telemetrytraces.TagAttributesV2TableName,
|
||||
telemetrytraces.SpanAttributesKeysTblName,
|
||||
telemetrytraces.SpanIndexV3TableName,
|
||||
telemetrymetrics.DBName,
|
||||
telemetrymetrics.AttributesMetadataTableName,
|
||||
telemetrymeter.DBName,
|
||||
telemetrymeter.SamplesAgg1dTableName,
|
||||
telemetrylogs.DBName,
|
||||
telemetrylogs.LogsV2TableName,
|
||||
telemetrylogs.TagAttributesV2TableName,
|
||||
telemetrylogs.LogAttributeKeysTblName,
|
||||
telemetrylogs.LogResourceKeysTblName,
|
||||
telemetryaudit.DBName,
|
||||
telemetryaudit.AuditLogsTableName,
|
||||
telemetryaudit.TagAttributesTableName,
|
||||
telemetryaudit.LogAttributeKeysTblName,
|
||||
telemetryaudit.LogResourceKeysTblName,
|
||||
telemetrymetadata.DBName,
|
||||
telemetrymetadata.AttributesMetadataTableName,
|
||||
telemetrymetadata.ColumnEvolutionMetadataTableName,
|
||||
flagger,
|
||||
)
|
||||
|
||||
global, err := factory.NewProviderFromNamedMap(
|
||||
ctx,
|
||||
providerSettings,
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
package auditstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
// NewFactory returns a provider factory for the audit statement builder. Its New
|
||||
// internalizes the FieldMapper, ConditionBuilder, and AggExprRewriter.
|
||||
func NewFactory(
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
fl flagger.Flagger,
|
||||
) factory.ProviderFactory[qbtypes.StatementBuilder[qbtypes.LogAggregation], statementbuilder.Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("audit"),
|
||||
func(_ context.Context, settings factory.ProviderSettings, _ statementbuilder.Config) (qbtypes.StatementBuilder[qbtypes.LogAggregation], error) {
|
||||
fm := audittelemetryschema.NewFieldMapper()
|
||||
cb := audittelemetryschema.NewConditionBuilder(fm)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(settings, audittelemetryschema.DefaultFullTextColumn, fm, cb, fl)
|
||||
return NewAuditQueryStatementBuilder(
|
||||
settings, metadataStore, fm, cb, aggExprRewriter, audittelemetryschema.DefaultFullTextColumn, fl,
|
||||
), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package statementbuilder
|
||||
|
||||
import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
)
|
||||
|
||||
// SkipResourceFingerprint configures when the resource fingerprint subquery is skipped in favor of main-table filtering.
|
||||
type SkipResourceFingerprint struct {
|
||||
Enabled bool `yaml:"enabled" mapstructure:"enabled"`
|
||||
// If count of fingerprint is above threshold, skip the fingerprint subquery and filter on main table instead.
|
||||
Threshold uint64 `yaml:"threshold" mapstructure:"threshold"`
|
||||
}
|
||||
|
||||
// Config represents the configuration for the statement builders.
|
||||
type Config struct {
|
||||
// SkipResourceFingerprint configures when the resource fingerprint subquery is skipped in favor of main-table filtering.
|
||||
SkipResourceFingerprint SkipResourceFingerprint `yaml:"skip_resource_fingerprint" mapstructure:"skip_resource_fingerprint"`
|
||||
}
|
||||
|
||||
// NewConfigFactory creates a new config factory for the statement builders.
|
||||
func NewConfigFactory() factory.ConfigFactory {
|
||||
return factory.NewConfigFactory(factory.MustNewName("statementbuilder"), newConfig)
|
||||
}
|
||||
|
||||
func newConfig() factory.Config {
|
||||
return Config{
|
||||
SkipResourceFingerprint: SkipResourceFingerprint{
|
||||
Enabled: false,
|
||||
Threshold: 100000,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Validate validates the configuration.
|
||||
func (c Config) Validate() error {
|
||||
if c.SkipResourceFingerprint.Enabled && c.SkipResourceFingerprint.Threshold == 0 {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "skip_resource_fingerprint.threshold must be > 0 when enabled")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
package logsstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
// NewFactory returns a provider factory for the logs statement builder. Its New
|
||||
// internalizes the FieldMapper, ConditionBuilder, and AggExprRewriter, and reads
|
||||
// SkipResourceFingerprint from the config.
|
||||
func NewFactory(
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
fl flagger.Flagger,
|
||||
) factory.ProviderFactory[qbtypes.StatementBuilder[qbtypes.LogAggregation], statementbuilder.Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("logs"),
|
||||
func(_ context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.StatementBuilder[qbtypes.LogAggregation], error) {
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(settings, logstelemetryschema.DefaultFullTextColumn, fm, cb, fl)
|
||||
return NewLogQueryStatementBuilder(
|
||||
settings, metadataStore, fm, cb, aggExprRewriter, logstelemetryschema.DefaultFullTextColumn,
|
||||
fl, telemetryStore, cfg.SkipResourceFingerprint.Enabled, cfg.SkipResourceFingerprint.Threshold,
|
||||
), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package meterstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
// NewFactory returns a provider factory for the meter statement builder. Its New
|
||||
// reuses the metrics FieldMapper/ConditionBuilder and delegates the final SELECT
|
||||
// to a metrics statement builder built via the metrics factory.
|
||||
func NewFactory(
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
fl flagger.Flagger,
|
||||
) factory.ProviderFactory[qbtypes.StatementBuilder[qbtypes.MetricAggregation], statementbuilder.Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("meter"),
|
||||
func(ctx context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.StatementBuilder[qbtypes.MetricAggregation], error) {
|
||||
metricsStatementBuilder, err := metricsstatementbuilder.NewFactory(metadataStore, fl).New(ctx, settings, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fm := metricstelemetryschema.NewFieldMapper()
|
||||
cb := metricstelemetryschema.NewConditionBuilder(fm)
|
||||
return NewMeterQueryStatementBuilder(settings, metadataStore, fm, cb, metricsStatementBuilder), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
package metricsstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
// NewFactory returns a provider factory for the metrics statement builder. Its
|
||||
// New internalizes the FieldMapper and ConditionBuilder and yields the concrete
|
||||
// *StatementBuilder so the meter builder can reuse it.
|
||||
func NewFactory(
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
fl flagger.Flagger,
|
||||
) factory.ProviderFactory[*StatementBuilder, statementbuilder.Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("metrics"),
|
||||
func(_ context.Context, settings factory.ProviderSettings, _ statementbuilder.Config) (*StatementBuilder, error) {
|
||||
fm := metricstelemetryschema.NewFieldMapper()
|
||||
cb := metricstelemetryschema.NewConditionBuilder(fm)
|
||||
return NewMetricQueryStatementBuilder(settings, metadataStore, fm, cb, fl), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
package statementbuilder
|
||||
|
||||
import (
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
)
|
||||
|
||||
// Builders bundles the per-signal statement builders that make up the query stack.
|
||||
//
|
||||
// This package is contract-only: it declares Builders (here) and Config (config.go)
|
||||
// and imports no sub-package. Each <signal>statementbuilder depends on this package
|
||||
// for Config and exposes its own factory.ProviderFactory[..., Config]; the composition
|
||||
// root (signoz.go::newQueryStack) is the only place that imports the concrete
|
||||
// sub-packages and assembles this struct. Keeping the edge as subs -> parent (never
|
||||
// the reverse) is what lets the sub-packages reference Config without an import cycle.
|
||||
type Builders struct {
|
||||
Trace qbtypes.StatementBuilder[qbtypes.TraceAggregation]
|
||||
TraceOperator qbtypes.TraceOperatorStatementBuilder
|
||||
Log qbtypes.StatementBuilder[qbtypes.LogAggregation]
|
||||
Audit qbtypes.StatementBuilder[qbtypes.LogAggregation]
|
||||
Metric qbtypes.StatementBuilder[qbtypes.MetricAggregation]
|
||||
Meter qbtypes.StatementBuilder[qbtypes.MetricAggregation]
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package tracesstatementbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
// NewFactory returns a provider factory for the trace query statement builder. Its
|
||||
// New internalizes the FieldMapper, ConditionBuilder, and AggExprRewriter, and reads
|
||||
// SkipResourceFingerprint from the config.
|
||||
func NewFactory(
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
fl flagger.Flagger,
|
||||
) factory.ProviderFactory[qbtypes.StatementBuilder[qbtypes.TraceAggregation], statementbuilder.Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("traces"),
|
||||
func(_ context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.StatementBuilder[qbtypes.TraceAggregation], error) {
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, fm, cb, fl)
|
||||
return NewTraceQueryStatementBuilder(
|
||||
settings, metadataStore, fm, cb, aggExprRewriter, telemetryStore, fl,
|
||||
cfg.SkipResourceFingerprint.Enabled, cfg.SkipResourceFingerprint.Threshold,
|
||||
), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// NewOperatorFactory returns a provider factory for the trace-operator statement
|
||||
// builder. The operator delegates sub-query construction to a trace query statement
|
||||
// builder, so it builds its own internally — mirroring how the meter factory builds
|
||||
// its own metrics builder.
|
||||
func NewOperatorFactory(
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
fl flagger.Flagger,
|
||||
) factory.ProviderFactory[qbtypes.TraceOperatorStatementBuilder, statementbuilder.Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("traceoperator"),
|
||||
func(_ context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.TraceOperatorStatementBuilder, error) {
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, fm, cb, fl)
|
||||
traceStmtBuilder := NewTraceQueryStatementBuilder(
|
||||
settings, metadataStore, fm, cb, aggExprRewriter, telemetryStore, fl,
|
||||
cfg.SkipResourceFingerprint.Enabled, cfg.SkipResourceFingerprint.Threshold,
|
||||
)
|
||||
return NewTraceOperatorStatementBuilder(
|
||||
settings, metadataStore, fm, cb, traceStmtBuilder, aggExprRewriter, fl,
|
||||
), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package audittelemetryschema
|
||||
package telemetryaudit
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package audittelemetryschema
|
||||
package telemetryaudit
|
||||
|
||||
import (
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
@@ -127,14 +127,3 @@ var auditLogColumns = map[string]*schema.Column{
|
||||
"scope_version": {Name: "scope_version", Type: schema.ColumnTypeString},
|
||||
"scope_string": {Name: "scope_string", Type: schema.MapColumnType{KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, ValueType: schema.ColumnTypeString}},
|
||||
}
|
||||
|
||||
const (
|
||||
DBName = "signoz_audit"
|
||||
AuditLogsTableName = "distributed_logs"
|
||||
AuditLogsLocalTableName = "logs"
|
||||
TagAttributesTableName = "distributed_tag_attributes"
|
||||
TagAttributesLocalTableName = "tag_attributes"
|
||||
LogAttributeKeysTblName = "distributed_logs_attribute_keys"
|
||||
LogResourceKeysTblName = "distributed_logs_resource_keys"
|
||||
LogsResourceTableName = "distributed_logs_resource"
|
||||
)
|
||||
@@ -1,4 +1,4 @@
|
||||
package audittelemetryschema
|
||||
package telemetryaudit
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package auditstatementbuilder
|
||||
package telemetryaudit
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,8 +10,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/resourcefilter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryresourcefilter"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -39,12 +38,12 @@ func NewAuditQueryStatementBuilder(
|
||||
fullTextColumn *telemetrytypes.TelemetryFieldKey,
|
||||
flagger flagger.Flagger,
|
||||
) *auditQueryStatementBuilder {
|
||||
auditSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema")
|
||||
auditSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryaudit")
|
||||
|
||||
resourceFilterStmtBuilder := resourcefilter.New[qbtypes.LogAggregation](
|
||||
resourceFilterStmtBuilder := telemetryresourcefilter.New[qbtypes.LogAggregation](
|
||||
settings,
|
||||
audittelemetryschema.DBName,
|
||||
audittelemetryschema.LogsResourceTableName,
|
||||
DBName,
|
||||
LogsResourceTableName,
|
||||
telemetrytypes.SignalLogs,
|
||||
telemetrytypes.SourceAudit,
|
||||
metadataStore,
|
||||
@@ -192,8 +191,8 @@ func (b *auditQueryStatementBuilder) adjustKeys(ctx context.Context, keys map[st
|
||||
}
|
||||
|
||||
func (b *auditQueryStatementBuilder) adjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) []string {
|
||||
if _, ok := audittelemetryschema.IntrinsicFields[key.Name]; ok {
|
||||
intrinsicField := audittelemetryschema.IntrinsicFields[key.Name]
|
||||
if _, ok := IntrinsicFields[key.Name]; ok {
|
||||
intrinsicField := IntrinsicFields[key.Name]
|
||||
return querybuilder.AdjustKey(key, keys, &intrinsicField)
|
||||
}
|
||||
return querybuilder.AdjustKey(key, keys, nil)
|
||||
@@ -220,26 +219,26 @@ func (b *auditQueryStatementBuilder) buildListQuery(
|
||||
cteArgs = append(cteArgs, args)
|
||||
}
|
||||
|
||||
sb.Select(audittelemetryschema.TimestampColumn)
|
||||
sb.SelectMore(audittelemetryschema.IDColumn)
|
||||
sb.Select(TimestampColumn)
|
||||
sb.SelectMore(IDColumn)
|
||||
if len(query.SelectFields) == 0 {
|
||||
sb.SelectMore(audittelemetryschema.TraceIDColumn)
|
||||
sb.SelectMore(audittelemetryschema.SpanIDColumn)
|
||||
sb.SelectMore(audittelemetryschema.TraceFlagsColumn)
|
||||
sb.SelectMore(audittelemetryschema.SeverityTextColumn)
|
||||
sb.SelectMore(audittelemetryschema.SeverityNumberColumn)
|
||||
sb.SelectMore(audittelemetryschema.ScopeNameColumn)
|
||||
sb.SelectMore(audittelemetryschema.ScopeVersionColumn)
|
||||
sb.SelectMore(audittelemetryschema.BodyColumn)
|
||||
sb.SelectMore(audittelemetryschema.EventNameColumn)
|
||||
sb.SelectMore(audittelemetryschema.AttributesStringColumn)
|
||||
sb.SelectMore(audittelemetryschema.AttributesNumberColumn)
|
||||
sb.SelectMore(audittelemetryschema.AttributesBoolColumn)
|
||||
sb.SelectMore(audittelemetryschema.ResourceColumn)
|
||||
sb.SelectMore(audittelemetryschema.ScopeStringColumn)
|
||||
sb.SelectMore(TraceIDColumn)
|
||||
sb.SelectMore(SpanIDColumn)
|
||||
sb.SelectMore(TraceFlagsColumn)
|
||||
sb.SelectMore(SeverityTextColumn)
|
||||
sb.SelectMore(SeverityNumberColumn)
|
||||
sb.SelectMore(ScopeNameColumn)
|
||||
sb.SelectMore(ScopeVersionColumn)
|
||||
sb.SelectMore(BodyColumn)
|
||||
sb.SelectMore(EventNameColumn)
|
||||
sb.SelectMore(AttributesStringColumn)
|
||||
sb.SelectMore(AttributesNumberColumn)
|
||||
sb.SelectMore(AttributesBoolColumn)
|
||||
sb.SelectMore(ResourceColumn)
|
||||
sb.SelectMore(ScopeStringColumn)
|
||||
} else {
|
||||
for index := range query.SelectFields {
|
||||
if query.SelectFields[index].Name == audittelemetryschema.TimestampColumn || query.SelectFields[index].Name == audittelemetryschema.IDColumn {
|
||||
if query.SelectFields[index].Name == TimestampColumn || query.SelectFields[index].Name == IDColumn {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -251,7 +250,7 @@ func (b *auditQueryStatementBuilder) buildListQuery(
|
||||
}
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", audittelemetryschema.DBName, audittelemetryschema.AuditLogsTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, AuditLogsTableName))
|
||||
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables)
|
||||
if err != nil {
|
||||
@@ -341,7 +340,7 @@ func (b *auditQueryStatementBuilder) buildTimeSeriesQuery(
|
||||
sb.SelectMore(fmt.Sprintf("%s AS __result_%d", rewritten, i))
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", audittelemetryschema.DBName, audittelemetryschema.AuditLogsTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, AuditLogsTableName))
|
||||
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables)
|
||||
if err != nil {
|
||||
@@ -479,7 +478,7 @@ func (b *auditQueryStatementBuilder) buildScalarQuery(
|
||||
}
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", audittelemetryschema.DBName, audittelemetryschema.AuditLogsTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, AuditLogsTableName))
|
||||
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables)
|
||||
if err != nil {
|
||||
@@ -1,4 +1,4 @@
|
||||
package auditstatementbuilder
|
||||
package telemetryaudit
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
||||
@@ -55,8 +54,8 @@ func newTestAuditStatementBuilder(t *testing.T) *auditQueryStatementBuilder {
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = auditFieldKeyMap()
|
||||
|
||||
fm := audittelemetryschema.NewFieldMapper()
|
||||
cb := audittelemetryschema.NewConditionBuilder(fm)
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
return NewAuditQueryStatementBuilder(
|
||||
@@ -65,7 +64,7 @@ func newTestAuditStatementBuilder(t *testing.T) *auditQueryStatementBuilder {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
audittelemetryschema.DefaultFullTextColumn,
|
||||
DefaultFullTextColumn,
|
||||
fl,
|
||||
)
|
||||
}
|
||||
12
pkg/telemetryaudit/tables.go
Normal file
12
pkg/telemetryaudit/tables.go
Normal file
@@ -0,0 +1,12 @@
|
||||
package telemetryaudit
|
||||
|
||||
const (
|
||||
DBName = "signoz_audit"
|
||||
AuditLogsTableName = "distributed_logs"
|
||||
AuditLogsLocalTableName = "logs"
|
||||
TagAttributesTableName = "distributed_tag_attributes"
|
||||
TagAttributesLocalTableName = "tag_attributes"
|
||||
LogAttributeKeysTblName = "distributed_logs_attribute_keys"
|
||||
LogResourceKeysTblName = "distributed_logs_resource_keys"
|
||||
LogsResourceTableName = "distributed_logs_resource"
|
||||
)
|
||||
@@ -1,4 +1,4 @@
|
||||
package logstelemetryschema
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package logstelemetryschema
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -548,7 +548,7 @@ func TestConditionForSynthesizedPrefixedKeys(t *testing.T) {
|
||||
t.Run("bare intrinsic column resolves to the column, not synthesized attributes", func(t *testing.T) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
key := telemetrytypes.TelemetryFieldKey{Name: "severity_text"}
|
||||
conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "ERROR", sb)
|
||||
conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "ERROR", sb)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, conds, 1)
|
||||
sb.Where(conds...)
|
||||
@@ -560,7 +560,7 @@ func TestConditionForSynthesizedPrefixedKeys(t *testing.T) {
|
||||
t.Run("attribute context", func(t *testing.T) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
key := telemetrytypes.TelemetryFieldKey{Name: "custom.key", FieldContext: telemetrytypes.FieldContextAttribute}
|
||||
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "v", sb)
|
||||
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "v", sb)
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, warnings)
|
||||
require.Len(t, conds, 2)
|
||||
@@ -571,7 +571,7 @@ func TestConditionForSynthesizedPrefixedKeys(t *testing.T) {
|
||||
t.Run("resource context", func(t *testing.T) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
key := telemetrytypes.TelemetryFieldKey{Name: "custom.key", FieldContext: telemetrytypes.FieldContextResource}
|
||||
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "v", sb)
|
||||
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "v", sb)
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, warnings)
|
||||
require.Len(t, conds, 2)
|
||||
@@ -582,7 +582,7 @@ func TestConditionForSynthesizedPrefixedKeys(t *testing.T) {
|
||||
t.Run("log context folds to attributes then body", func(t *testing.T) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
key := telemetrytypes.TelemetryFieldKey{Name: "custom.key", FieldContext: telemetrytypes.FieldContextLog}
|
||||
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "v", sb)
|
||||
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "v", sb)
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, warnings)
|
||||
require.Len(t, conds, 4)
|
||||
@@ -1,6 +1,8 @@
|
||||
package logstelemetryschema
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/SigNoz/signoz-otel-collector/constants"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -39,8 +41,9 @@ const (
|
||||
|
||||
// messageSubColumn is the ClickHouse sub-column that body searches map to
|
||||
// when use_json_body feature flag is true.
|
||||
messageSubField = "message"
|
||||
messageSubColumn = "body_v2.message"
|
||||
messageSubField = "message"
|
||||
messageSubColumn = "body_v2.message"
|
||||
bodySearchDefaultWarning = "body searches default to `body.message:string`. Use `body.<key>` to search a different field inside body"
|
||||
|
||||
// bodyMessageField is the field name addressing the message sub-field of the
|
||||
// body when use_json_body is enabled (i.e. `body` + `.` + `message`). hasToken
|
||||
@@ -134,16 +137,10 @@ var (
|
||||
}
|
||||
)
|
||||
|
||||
const (
|
||||
DBName = "signoz_logs"
|
||||
LogsV2TableName = "distributed_logs_v2"
|
||||
LogsV2LocalTableName = "logs_v2"
|
||||
TagAttributesV2TableName = "distributed_tag_attributes_v2"
|
||||
TagAttributesV2LocalTableName = "tag_attributes_v2"
|
||||
LogAttributeKeysTblName = "distributed_logs_attribute_keys"
|
||||
LogResourceKeysTblName = "distributed_logs_resource_keys"
|
||||
LogsResourceV2TableName = "distributed_logs_v2_resource"
|
||||
PathTypesTableName = "distributed_json_path_types"
|
||||
PromotedPathsTableName = "distributed_json_promoted_paths"
|
||||
SkipIndexTableName = "system.data_skipping_indices"
|
||||
)
|
||||
func bodyAliasExpression(bodyJSONEnabled bool) string {
|
||||
if !bodyJSONEnabled {
|
||||
return LogsV2BodyColumn
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s as body", LogsV2BodyV2Column)
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package logstelemetryschema
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package logstelemetryschema
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package logstelemetryschema
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -19,7 +19,7 @@ func TestLikeAndILikeWithoutWildcards_Warns(t *testing.T) {
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keys := BuildCompleteFieldKeyMap(releaseTime)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
@@ -57,7 +57,7 @@ func TestLikeAndILikeWithWildcards_NoWarn(t *testing.T) {
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keys := BuildCompleteFieldKeyMap(releaseTime)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: context.Background(),
|
||||
@@ -1,4 +1,4 @@
|
||||
package logstelemetryschema
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -21,7 +21,7 @@ func TestFilterExprLogsBodyJSON(t *testing.T) {
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
// Define a comprehensive set of field keys to support all test cases
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keys := BuildCompleteFieldKeyMap(releaseTime)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: context.Background(),
|
||||
@@ -1,4 +1,4 @@
|
||||
package logstelemetryschema
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -41,7 +41,7 @@ func TestFilterExprLogs(t *testing.T) {
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
// Define a comprehensive set of field keys to support all test cases
|
||||
keys := BuildCompleteFieldKeyMap(releaseTime)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
@@ -2476,7 +2476,7 @@ func TestFilterExprLogsConflictNegation(t *testing.T) {
|
||||
|
||||
// Define a comprehensive set of field keys to support all test cases
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keys := BuildCompleteFieldKeyMap(releaseTime)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
keys["body"] = []*telemetrytypes.TelemetryFieldKey{
|
||||
{
|
||||
@@ -1,4 +1,4 @@
|
||||
package logsstatementbuilder
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
||||
@@ -44,14 +43,14 @@ func TestStatementBuilderGroupByUnknownKey(t *testing.T) {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
fl := flaggertest.WithUseJSONBody(t, c.useJSONBody)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
statementBuilder := NewLogQueryStatementBuilder(
|
||||
instrumentationtest.New().ToProviderSettings(),
|
||||
mockMetadataStore, fm, cb, aggExprRewriter,
|
||||
logstelemetryschema.DefaultFullTextColumn, fl, nil, false, 100000,
|
||||
DefaultFullTextColumn, fl, nil, false, 100000,
|
||||
)
|
||||
|
||||
query := qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
||||
@@ -1,4 +1,4 @@
|
||||
package logstelemetryschema
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -1,4 +1,4 @@
|
||||
package logstelemetryschema
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"strings"
|
||||
@@ -1,4 +1,4 @@
|
||||
package logsstatementbuilder
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
||||
@@ -1226,7 +1225,7 @@ func TestJSONStmtBuilder_OrderBy(t *testing.T) {
|
||||
func TestResourceAggrAndGroupBy_WithJSONEnabled(t *testing.T) {
|
||||
statementBuilder, metadataStore := buildJSONTestStatementBuilder(t, false)
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keysMap := logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
keysMap := buildCompleteFieldKeyMap(releaseTime)
|
||||
for _, keys := range keysMap {
|
||||
for _, key := range keys {
|
||||
metadataStore.SetKey(key)
|
||||
@@ -1288,7 +1287,7 @@ func TestResourceAggrAndGroupBy_WithJSONEnabled(t *testing.T) {
|
||||
|
||||
func buildTestTelemetryMetadataStore(t *testing.T, addIndexes bool) *telemetrytypestest.MockMetadataStore {
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.SetStaticFields(logstelemetryschema.IntrinsicFields)
|
||||
mockMetadataStore.SetStaticFields(IntrinsicFields)
|
||||
types, _ := telemetrytypes.TestJSONTypeSet()
|
||||
for path, fieldDataTypes := range types {
|
||||
for _, fdt := range fieldDataTypes {
|
||||
@@ -1308,15 +1307,15 @@ func buildTestTelemetryMetadataStore(t *testing.T, addIndexes bool) *telemetryty
|
||||
Name: path,
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
FieldDataType: fdt,
|
||||
BaseColumn: logstelemetryschema.LogsV2BodyV2Column,
|
||||
IndexExpression: schemamigrator.JSONSubColumnIndexExpr(logstelemetryschema.LogsV2BodyV2Column, path, jsonType.StringValue()),
|
||||
BaseColumn: LogsV2BodyV2Column,
|
||||
IndexExpression: schemamigrator.JSONSubColumnIndexExpr(LogsV2BodyV2Column, path, jsonType.StringValue()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
err := key.SetJSONAccessPlan(telemetrytypes.JSONColumnMetadata{
|
||||
BaseColumn: logstelemetryschema.LogsV2BodyV2Column,
|
||||
PromotedColumn: logstelemetryschema.LogsV2BodyPromotedColumn,
|
||||
BaseColumn: LogsV2BodyV2Column,
|
||||
PromotedColumn: LogsV2BodyPromotedColumn,
|
||||
}, types)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -1332,8 +1331,8 @@ func buildJSONTestStatementBuilder(t *testing.T, addIndexes bool) (*logQueryStat
|
||||
|
||||
mockMetadataStore := buildTestTelemetryMetadataStore(t, addIndexes)
|
||||
fl := flaggertest.WithUseJSONBody(t, true)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
@@ -1343,7 +1342,7 @@ func buildJSONTestStatementBuilder(t *testing.T, addIndexes bool) (*logQueryStat
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -1,4 +1,4 @@
|
||||
package logstelemetryschema
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package logsstatementbuilder
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,8 +10,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/resourcefilter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryresourcefilter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -20,22 +19,12 @@ import (
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
)
|
||||
|
||||
const bodySearchDefaultWarning = "body searches default to `body.message:string`. Use `body.<key>` to search a different field inside body"
|
||||
|
||||
func bodyAliasExpression(bodyJSONEnabled bool) string {
|
||||
if !bodyJSONEnabled {
|
||||
return logstelemetryschema.LogsV2BodyColumn
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s as body", logstelemetryschema.LogsV2BodyV2Column)
|
||||
}
|
||||
|
||||
type logQueryStatementBuilder struct {
|
||||
logger *slog.Logger
|
||||
metadataStore telemetrytypes.MetadataStore
|
||||
fm qbtypes.FieldMapper
|
||||
cb qbtypes.ConditionBuilder
|
||||
resourceFilterResolver *resourcefilter.ResourceFingerprintResolver[qbtypes.LogAggregation]
|
||||
resourceFilterResolver *telemetryresourcefilter.ResourceFingerprintResolver[qbtypes.LogAggregation]
|
||||
aggExprRewriter qbtypes.AggExprRewriter
|
||||
fl flagger.Flagger
|
||||
skipResourceFingerprintEnabled bool
|
||||
@@ -57,12 +46,12 @@ func NewLogQueryStatementBuilder(
|
||||
skipResourceFingerprintEnable bool,
|
||||
skipResourceFingerprintThreshold uint64,
|
||||
) *logQueryStatementBuilder {
|
||||
logsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema")
|
||||
logsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrylogs")
|
||||
|
||||
resourceFilterResolver := resourcefilter.NewResolver[qbtypes.LogAggregation](
|
||||
resourceFilterResolver := telemetryresourcefilter.NewResolver[qbtypes.LogAggregation](
|
||||
settings,
|
||||
logstelemetryschema.DBName,
|
||||
logstelemetryschema.LogsResourceV2TableName,
|
||||
DBName,
|
||||
LogsResourceV2TableName,
|
||||
telemetrytypes.SignalLogs,
|
||||
telemetrytypes.SourceUnspecified,
|
||||
metadataStore,
|
||||
@@ -185,7 +174,7 @@ func getKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation], bo
|
||||
// TODO(Piyush): Setup better for coming FTS support.
|
||||
if bodyJSONEnabled {
|
||||
for _, sel := range keySelectors {
|
||||
if sel.Name == logstelemetryschema.LogsV2BodyColumn {
|
||||
if sel.Name == LogsV2BodyColumn {
|
||||
warnings = append(warnings, bodySearchDefaultWarning)
|
||||
break
|
||||
}
|
||||
@@ -261,8 +250,8 @@ func (b *logQueryStatementBuilder) adjustKeys(ctx context.Context, keys map[stri
|
||||
func (b *logQueryStatementBuilder) adjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) []string {
|
||||
// First check if it matches with any intrinsic fields
|
||||
var intrinsicOrCalculatedField telemetrytypes.TelemetryFieldKey
|
||||
if _, ok := logstelemetryschema.IntrinsicFields[key.Name]; ok {
|
||||
intrinsicOrCalculatedField = logstelemetryschema.IntrinsicFields[key.Name]
|
||||
if _, ok := IntrinsicFields[key.Name]; ok {
|
||||
intrinsicOrCalculatedField = IntrinsicFields[key.Name]
|
||||
return querybuilder.AdjustKey(key, keys, &intrinsicOrCalculatedField)
|
||||
}
|
||||
|
||||
@@ -295,28 +284,28 @@ func (b *logQueryStatementBuilder) buildListQuery(
|
||||
}
|
||||
|
||||
// Select timestamp and id by default
|
||||
sb.Select(logstelemetryschema.LogsV2TimestampColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2IDColumn)
|
||||
sb.Select(LogsV2TimestampColumn)
|
||||
sb.SelectMore(LogsV2IDColumn)
|
||||
if len(query.SelectFields) == 0 {
|
||||
// Select all default columns
|
||||
sb.SelectMore(logstelemetryschema.LogsV2TraceIDColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2SpanIDColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2TraceFlagsColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2SeverityTextColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2SeverityNumberColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2ScopeNameColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2ScopeVersionColumn)
|
||||
sb.SelectMore(LogsV2TraceIDColumn)
|
||||
sb.SelectMore(LogsV2SpanIDColumn)
|
||||
sb.SelectMore(LogsV2TraceFlagsColumn)
|
||||
sb.SelectMore(LogsV2SeverityTextColumn)
|
||||
sb.SelectMore(LogsV2SeverityNumberColumn)
|
||||
sb.SelectMore(LogsV2ScopeNameColumn)
|
||||
sb.SelectMore(LogsV2ScopeVersionColumn)
|
||||
sb.SelectMore(bodyAliasExpression(b.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID))))
|
||||
sb.SelectMore(logstelemetryschema.LogsV2AttributesStringColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2AttributesNumberColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2AttributesBoolColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2ResourcesStringColumn)
|
||||
sb.SelectMore(logstelemetryschema.LogsV2ScopeStringColumn)
|
||||
sb.SelectMore(LogsV2AttributesStringColumn)
|
||||
sb.SelectMore(LogsV2AttributesNumberColumn)
|
||||
sb.SelectMore(LogsV2AttributesBoolColumn)
|
||||
sb.SelectMore(LogsV2ResourcesStringColumn)
|
||||
sb.SelectMore(LogsV2ScopeStringColumn)
|
||||
|
||||
} else {
|
||||
// Select specified columns
|
||||
for index := range query.SelectFields {
|
||||
if query.SelectFields[index].Name == logstelemetryschema.LogsV2TimestampColumn || query.SelectFields[index].Name == logstelemetryschema.LogsV2IDColumn {
|
||||
if query.SelectFields[index].Name == LogsV2TimestampColumn || query.SelectFields[index].Name == LogsV2IDColumn {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -329,7 +318,7 @@ func (b *logQueryStatementBuilder) buildListQuery(
|
||||
}
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", logstelemetryschema.DBName, logstelemetryschema.LogsV2TableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
|
||||
// Add filter conditions
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables, skipResourceFilter)
|
||||
|
||||
@@ -435,7 +424,7 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
|
||||
}
|
||||
|
||||
// Add FROM clause
|
||||
sb.From(fmt.Sprintf("%s.%s", logstelemetryschema.DBName, logstelemetryschema.LogsV2TableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
|
||||
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables, skipResourceFilter)
|
||||
|
||||
@@ -603,7 +592,7 @@ func (b *logQueryStatementBuilder) buildScalarQuery(
|
||||
}
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", logstelemetryschema.DBName, logstelemetryschema.LogsV2TableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
|
||||
|
||||
// Add filter conditions
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables, skipResourceFilter)
|
||||
@@ -1,4 +1,4 @@
|
||||
package logstelemetryschema
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"testing"
|
||||
@@ -1,4 +1,4 @@
|
||||
package logsstatementbuilder
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -213,12 +212,12 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
fl := flaggertest.New(t)
|
||||
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
keysMap := logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
keysMap := buildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
mockMetadataStore.KeysMap = keysMap
|
||||
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
@@ -228,7 +227,7 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -356,12 +355,12 @@ func TestStatementBuilderListQuery(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fl := flaggertest.New(t)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
fm := NewFieldMapper(fl)
|
||||
|
||||
// Create a test release time
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
@@ -371,7 +370,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -508,11 +507,11 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fl := flaggertest.New(t)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
fm := NewFieldMapper(fl)
|
||||
// Create a test release time
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
@@ -522,7 +521,7 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -586,11 +585,11 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fl := flaggertest.New(t)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
fm := NewFieldMapper(fl)
|
||||
// Create a test release time
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
@@ -600,7 +599,7 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -685,9 +684,9 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
fl := flaggertest.New(t)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMapCollision()
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
fm := NewFieldMapper(fl)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMapCollision()
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
@@ -697,7 +696,7 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -741,8 +740,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: logstelemetryschema.IntrinsicFields["severity_text"],
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: IntrinsicFields["severity_text"],
|
||||
},
|
||||
{
|
||||
name: "intrinsic field with other key match - no override",
|
||||
@@ -778,7 +777,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "severity_number",
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
@@ -792,8 +791,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["service.name"][0],
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["service.name"][0],
|
||||
},
|
||||
{
|
||||
name: "single matching key with incorrect context specified - no override",
|
||||
@@ -802,7 +801,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
@@ -816,8 +815,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["service.name"][0],
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["service.name"][0],
|
||||
},
|
||||
{
|
||||
name: "multiple matching keys - all materialized",
|
||||
@@ -826,7 +825,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "multi.mat.key",
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
@@ -840,7 +839,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "mixed.materialization.key",
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
@@ -854,8 +853,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["mixed.materialization.key"][0],
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["mixed.materialization.key"][0],
|
||||
},
|
||||
{
|
||||
name: "no matching keys - unknown field",
|
||||
@@ -864,7 +863,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "unknown.field",
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
@@ -879,7 +878,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "unknown.field",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
@@ -894,8 +893,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["mat.key"][0],
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["mat.key"][0],
|
||||
},
|
||||
{
|
||||
name: "non-materialized field",
|
||||
@@ -904,16 +903,16 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["user.id"][0],
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["user.id"][0],
|
||||
},
|
||||
}
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
fm := NewFieldMapper(fl)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMapCollision()
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMapCollision()
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
@@ -923,7 +922,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -1055,11 +1054,11 @@ func TestStmtBuilderBodyField(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
fl := flaggertest.WithUseJSONBody(t, c.enableUseJSONBody)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
// build the key map
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
for _, field := range logstelemetryschema.IntrinsicFields {
|
||||
for _, field := range IntrinsicFields {
|
||||
f := field
|
||||
mockMetadataStore.KeysMap[field.Name] = append(mockMetadataStore.KeysMap[field.Name], &f)
|
||||
}
|
||||
@@ -1070,7 +1069,7 @@ func TestStmtBuilderBodyField(t *testing.T) {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -1156,11 +1155,11 @@ func TestStmtBuilderBodyFullTextSearch(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
fl := flaggertest.WithUseJSONBody(t, c.enableUseJSONBody)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
// build the key map
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
for _, field := range logstelemetryschema.IntrinsicFields {
|
||||
for _, field := range IntrinsicFields {
|
||||
f := field
|
||||
mockMetadataStore.KeysMap[field.Name] = append(mockMetadataStore.KeysMap[field.Name], &f)
|
||||
}
|
||||
@@ -1171,7 +1170,7 @@ func TestStmtBuilderBodyFullTextSearch(t *testing.T) {
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
DefaultFullTextColumn,
|
||||
fl,
|
||||
nil,
|
||||
false,
|
||||
@@ -1274,14 +1273,14 @@ func newSkipResourceFingerprintLogsBuilder(
|
||||
t.Helper()
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := logstelemetryschema.NewFieldMapper(fl)
|
||||
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMap(time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC))
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC))
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(
|
||||
instrumentationtest.New().ToProviderSettings(),
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
DefaultFullTextColumn,
|
||||
fm,
|
||||
cb,
|
||||
fl,
|
||||
@@ -1293,7 +1292,7 @@ func newSkipResourceFingerprintLogsBuilder(
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
logstelemetryschema.DefaultFullTextColumn,
|
||||
DefaultFullTextColumn,
|
||||
fl,
|
||||
telemetryStore,
|
||||
skipEnable,
|
||||
15
pkg/telemetrylogs/tables.go
Normal file
15
pkg/telemetrylogs/tables.go
Normal file
@@ -0,0 +1,15 @@
|
||||
package telemetrylogs
|
||||
|
||||
const (
|
||||
DBName = "signoz_logs"
|
||||
LogsV2TableName = "distributed_logs_v2"
|
||||
LogsV2LocalTableName = "logs_v2"
|
||||
TagAttributesV2TableName = "distributed_tag_attributes_v2"
|
||||
TagAttributesV2LocalTableName = "tag_attributes_v2"
|
||||
LogAttributeKeysTblName = "distributed_logs_attribute_keys"
|
||||
LogResourceKeysTblName = "distributed_logs_resource_keys"
|
||||
LogsResourceV2TableName = "distributed_logs_v2_resource"
|
||||
PathTypesTableName = "distributed_json_path_types"
|
||||
PromotedPathsTableName = "distributed_json_promoted_paths"
|
||||
SkipIndexTableName = "system.data_skipping_indices"
|
||||
)
|
||||
@@ -1,4 +1,4 @@
|
||||
package logstelemetryschema
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"strings"
|
||||
@@ -19,7 +19,7 @@ func limitString(s string, maxLen int) string {
|
||||
}
|
||||
|
||||
// Function to build a complete field key map for testing all scenarios.
|
||||
func BuildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytypes.TelemetryFieldKey {
|
||||
func buildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytypes.TelemetryFieldKey {
|
||||
keysMap := map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
"service.name": {
|
||||
{
|
||||
@@ -953,7 +953,7 @@ func BuildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytype
|
||||
return keysMap
|
||||
}
|
||||
|
||||
func BuildCompleteFieldKeyMapCollision() map[string][]*telemetrytypes.TelemetryFieldKey {
|
||||
func buildCompleteFieldKeyMapCollision() map[string][]*telemetrytypes.TelemetryFieldKey {
|
||||
keysMap := map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
"service.name": {
|
||||
{
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"github.com/SigNoz/signoz-otel-collector/constants"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -139,8 +139,8 @@ func buildListLogsJSONIndexesQuery(cluster string, filters ...string) (string, [
|
||||
"name", "type_full", "expr", "granularity",
|
||||
).From(fmt.Sprintf("clusterAllReplicas('%s', %s)", cluster, SkipIndexTableName))
|
||||
|
||||
sb.Where(sb.Equal("database", logstelemetryschema.DBName))
|
||||
sb.Where(sb.Equal("table", logstelemetryschema.LogsV2LocalTableName))
|
||||
sb.Where(sb.Equal("database", telemetrylogs.DBName))
|
||||
sb.Where(sb.Equal("table", telemetrylogs.LogsV2LocalTableName))
|
||||
sb.Where(sb.Or(
|
||||
sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyV2ColumnPrefix))),
|
||||
sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyPromotedColumnPrefix))),
|
||||
@@ -189,12 +189,12 @@ func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ..
|
||||
baseColumn := ""
|
||||
fieldName := ""
|
||||
switch {
|
||||
case strings.HasPrefix(columnExpr, logstelemetryschema.BodyV2ColumnPrefix):
|
||||
baseColumn = logstelemetryschema.BodyV2ColumnPrefix
|
||||
fieldName = strings.TrimPrefix(columnExpr, logstelemetryschema.BodyV2ColumnPrefix)
|
||||
case strings.HasPrefix(columnExpr, logstelemetryschema.BodyPromotedColumnPrefix):
|
||||
baseColumn = logstelemetryschema.BodyPromotedColumnPrefix
|
||||
fieldName = strings.TrimPrefix(columnExpr, logstelemetryschema.BodyPromotedColumnPrefix)
|
||||
case strings.HasPrefix(columnExpr, telemetrylogs.BodyV2ColumnPrefix):
|
||||
baseColumn = telemetrylogs.BodyV2ColumnPrefix
|
||||
fieldName = strings.TrimPrefix(columnExpr, telemetrylogs.BodyV2ColumnPrefix)
|
||||
case strings.HasPrefix(columnExpr, telemetrylogs.BodyPromotedColumnPrefix):
|
||||
baseColumn = telemetrylogs.BodyPromotedColumnPrefix
|
||||
fieldName = strings.TrimPrefix(columnExpr, telemetrylogs.BodyPromotedColumnPrefix)
|
||||
}
|
||||
fieldName = strings.ReplaceAll(fieldName, "`", "")
|
||||
|
||||
@@ -228,12 +228,12 @@ func (t *telemetryMetaStore) ListJSONValues(ctx context.Context, path string, li
|
||||
}
|
||||
|
||||
if promoted {
|
||||
path = logstelemetryschema.BodyPromotedColumnPrefix + path
|
||||
path = telemetrylogs.BodyPromotedColumnPrefix + path
|
||||
} else {
|
||||
path = logstelemetryschema.BodyV2ColumnPrefix + path
|
||||
path = telemetrylogs.BodyV2ColumnPrefix + path
|
||||
}
|
||||
|
||||
from := fmt.Sprintf("%s.%s", logstelemetryschema.DBName, logstelemetryschema.LogsV2TableName)
|
||||
from := fmt.Sprintf("%s.%s", telemetrylogs.DBName, telemetrylogs.LogsV2TableName)
|
||||
colExpr := func(typ telemetrytypes.JSONDataType) string {
|
||||
return fmt.Sprintf("dynamicElement(%s, '%s')", path, typ.StringValue())
|
||||
}
|
||||
@@ -382,7 +382,7 @@ func (t *telemetryMetaStore) IsPathPromoted(ctx context.Context, path string) (b
|
||||
split := strings.Split(path, telemetrytypes.ArraySep)
|
||||
pathSegment := split[0]
|
||||
query := fmt.Sprintf("SELECT 1 FROM %s.%s WHERE signal = ? AND column_name = ? AND field_context = ? AND field_name = ? LIMIT 1", DBName, PromotedPathsTableName)
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, telemetrytypes.SignalLogs, logstelemetryschema.LogsV2BodyPromotedColumn, telemetrytypes.FieldContextBody, pathSegment)
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, telemetrytypes.SignalLogs, telemetrylogs.LogsV2BodyPromotedColumn, telemetrytypes.FieldContextBody, pathSegment)
|
||||
if err != nil {
|
||||
return false, errors.WrapInternalf(err, CodeFailCheckPathPromoted, "failed to check if path %s is promoted", path)
|
||||
}
|
||||
@@ -397,7 +397,7 @@ func (t *telemetryMetaStore) GetPromotedPaths(ctx context.Context, paths ...stri
|
||||
sb := sqlbuilder.Select("field_name").From(fmt.Sprintf("%s.%s", DBName, PromotedPathsTableName))
|
||||
conditions := []string{
|
||||
sb.Equal("signal", telemetrytypes.SignalLogs),
|
||||
sb.Equal("column_name", logstelemetryschema.LogsV2BodyPromotedColumn),
|
||||
sb.Equal("column_name", telemetrylogs.LogsV2BodyPromotedColumn),
|
||||
sb.Equal("field_context", telemetrytypes.FieldContextBody),
|
||||
sb.NotEqual("field_name", "__all__"),
|
||||
}
|
||||
@@ -433,8 +433,8 @@ func (t *telemetryMetaStore) GetPromotedPaths(ctx context.Context, paths ...stri
|
||||
// TODO(Piyush): Remove this function.
|
||||
func CleanPathPrefixes(path string) string {
|
||||
path = strings.TrimPrefix(path, telemetrytypes.BodyJSONStringSearchPrefix)
|
||||
path = strings.TrimPrefix(path, logstelemetryschema.BodyV2ColumnPrefix)
|
||||
path = strings.TrimPrefix(path, logstelemetryschema.BodyPromotedColumnPrefix)
|
||||
path = strings.TrimPrefix(path, telemetrylogs.BodyV2ColumnPrefix)
|
||||
path = strings.TrimPrefix(path, telemetrylogs.BodyPromotedColumnPrefix)
|
||||
return path
|
||||
}
|
||||
|
||||
@@ -454,7 +454,7 @@ func (t *telemetryMetaStore) PromotePaths(ctx context.Context, paths ...string)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
if err := batch.Append(telemetrytypes.SignalLogs, logstelemetryschema.LogsV2BodyPromotedColumn, "JSON()", telemetrytypes.FieldContextBody, trimmed, 0, releaseTime); err != nil {
|
||||
if err := batch.Append(telemetrytypes.SignalLogs, telemetrylogs.LogsV2BodyPromotedColumn, "JSON()", telemetrytypes.FieldContextBody, trimmed, 0, releaseTime); err != nil {
|
||||
_ = batch.Abort()
|
||||
return errors.WrapInternalf(err, CodeFailedToAppendPath, "failed to append path")
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz-otel-collector/constants"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -25,8 +25,8 @@ func TestBuildListLogsJSONIndexesQuery(t *testing.T) {
|
||||
expectedSQL: "SELECT name, type_full, expr, granularity FROM clusterAllReplicas('test-cluster', system.data_skipping_indices) " +
|
||||
"WHERE database = ? AND table = ? AND (LOWER(expr) LIKE LOWER(?) OR LOWER(expr) LIKE LOWER(?))",
|
||||
expectedArgs: []any{
|
||||
logstelemetryschema.DBName,
|
||||
logstelemetryschema.LogsV2LocalTableName,
|
||||
telemetrylogs.DBName,
|
||||
telemetrylogs.LogsV2LocalTableName,
|
||||
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyV2ColumnPrefix)),
|
||||
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyPromotedColumnPrefix)),
|
||||
},
|
||||
@@ -38,8 +38,8 @@ func TestBuildListLogsJSONIndexesQuery(t *testing.T) {
|
||||
expectedSQL: "SELECT name, type_full, expr, granularity FROM clusterAllReplicas('test-cluster', system.data_skipping_indices) " +
|
||||
"WHERE database = ? AND table = ? AND (LOWER(expr) LIKE LOWER(?) OR LOWER(expr) LIKE LOWER(?)) AND (LOWER(replaceAll(expr, '`', '')) LIKE LOWER(?) OR LOWER(replaceAll(expr, '`', '')) LIKE LOWER(?))",
|
||||
expectedArgs: []any{
|
||||
logstelemetryschema.DBName,
|
||||
logstelemetryschema.LogsV2LocalTableName,
|
||||
telemetrylogs.DBName,
|
||||
telemetrylogs.LogsV2LocalTableName,
|
||||
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyV2ColumnPrefix)),
|
||||
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyPromotedColumnPrefix)),
|
||||
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains("foo")),
|
||||
|
||||
@@ -14,12 +14,11 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryaudit"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
@@ -78,6 +77,27 @@ func escapeForLike(s string) string {
|
||||
func NewTelemetryMetaStore(
|
||||
settings factory.ProviderSettings,
|
||||
telemetrystore telemetrystore.TelemetryStore,
|
||||
tracesDBName string,
|
||||
tracesFieldsTblName string,
|
||||
spanAttributesKeysTblName string,
|
||||
indexV3TblName string,
|
||||
metricsDBName string,
|
||||
metricsFieldsTblName string,
|
||||
meterDBName string,
|
||||
meterFieldsTblName string,
|
||||
logsDBName string,
|
||||
logsV2TblName string,
|
||||
logsFieldsTblName string,
|
||||
logAttributeKeysTblName string,
|
||||
logResourceKeysTblName string,
|
||||
auditDBName string,
|
||||
auditLogsTblName string,
|
||||
auditFieldsTblName string,
|
||||
auditAttributeKeysTblName string,
|
||||
auditResourceKeysTblName string,
|
||||
relatedMetadataDBName string,
|
||||
relatedMetadataTblName string,
|
||||
columnEvolutionMetadataTblName string,
|
||||
fl flagger.Flagger,
|
||||
) telemetrytypes.MetadataStore {
|
||||
metadataSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrymetadata")
|
||||
@@ -88,32 +108,32 @@ func NewTelemetryMetaStore(
|
||||
t := &telemetryMetaStore{
|
||||
logger: metadataSettings.Logger(),
|
||||
telemetrystore: telemetrystore,
|
||||
tracesDBName: tracestelemetryschema.DBName,
|
||||
tracesFieldsTblName: tracestelemetryschema.TagAttributesV2TableName,
|
||||
spanAttributesKeysTblName: tracestelemetryschema.SpanAttributesKeysTblName,
|
||||
indexV3TblName: tracestelemetryschema.SpanIndexV3TableName,
|
||||
metricsDBName: metricstelemetryschema.DBName,
|
||||
metricsFieldsTblName: metricstelemetryschema.AttributesMetadataTableName,
|
||||
meterDBName: metertelemetryschema.DBName,
|
||||
meterFieldsTblName: metertelemetryschema.SamplesAgg1dTableName,
|
||||
logsDBName: logstelemetryschema.DBName,
|
||||
logsV2TblName: logstelemetryschema.LogsV2TableName,
|
||||
logsFieldsTblName: logstelemetryschema.TagAttributesV2TableName,
|
||||
logAttributeKeysTblName: logstelemetryschema.LogAttributeKeysTblName,
|
||||
logResourceKeysTblName: logstelemetryschema.LogResourceKeysTblName,
|
||||
auditDBName: audittelemetryschema.DBName,
|
||||
auditLogsTblName: audittelemetryschema.AuditLogsTableName,
|
||||
auditFieldsTblName: audittelemetryschema.TagAttributesTableName,
|
||||
auditAttributeKeysTblName: audittelemetryschema.LogAttributeKeysTblName,
|
||||
auditResourceKeysTblName: audittelemetryschema.LogResourceKeysTblName,
|
||||
relatedMetadataDBName: DBName,
|
||||
relatedMetadataTblName: AttributesMetadataTableName,
|
||||
columnEvolutionMetadataTblName: ColumnEvolutionMetadataTableName,
|
||||
tracesDBName: tracesDBName,
|
||||
tracesFieldsTblName: tracesFieldsTblName,
|
||||
spanAttributesKeysTblName: spanAttributesKeysTblName,
|
||||
indexV3TblName: indexV3TblName,
|
||||
metricsDBName: metricsDBName,
|
||||
metricsFieldsTblName: metricsFieldsTblName,
|
||||
meterDBName: meterDBName,
|
||||
meterFieldsTblName: meterFieldsTblName,
|
||||
logsDBName: logsDBName,
|
||||
logsV2TblName: logsV2TblName,
|
||||
logsFieldsTblName: logsFieldsTblName,
|
||||
logAttributeKeysTblName: logAttributeKeysTblName,
|
||||
logResourceKeysTblName: logResourceKeysTblName,
|
||||
auditDBName: auditDBName,
|
||||
auditLogsTblName: auditLogsTblName,
|
||||
auditFieldsTblName: auditFieldsTblName,
|
||||
auditAttributeKeysTblName: auditAttributeKeysTblName,
|
||||
auditResourceKeysTblName: auditResourceKeysTblName,
|
||||
relatedMetadataDBName: relatedMetadataDBName,
|
||||
relatedMetadataTblName: relatedMetadataTblName,
|
||||
columnEvolutionMetadataTblName: columnEvolutionMetadataTblName,
|
||||
jsonColumnMetadata: map[telemetrytypes.Signal]map[telemetrytypes.FieldContext]telemetrytypes.JSONColumnMetadata{
|
||||
telemetrytypes.SignalLogs: {
|
||||
telemetrytypes.FieldContextBody: telemetrytypes.JSONColumnMetadata{
|
||||
BaseColumn: logstelemetryschema.LogsV2BodyV2Column,
|
||||
PromotedColumn: logstelemetryschema.LogsV2BodyPromotedColumn,
|
||||
BaseColumn: telemetrylogs.LogsV2BodyV2Column,
|
||||
PromotedColumn: telemetrylogs.LogsV2BodyPromotedColumn,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -289,8 +309,8 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
|
||||
complete := rowCount <= limit
|
||||
|
||||
staticKeys := []string{"isRoot", "isEntryPoint"}
|
||||
staticKeys = append(staticKeys, maps.Keys(tracestelemetryschema.IntrinsicFields)...)
|
||||
staticKeys = append(staticKeys, maps.Keys(tracestelemetryschema.CalculatedFields)...)
|
||||
staticKeys = append(staticKeys, maps.Keys(telemetrytraces.IntrinsicFields)...)
|
||||
staticKeys = append(staticKeys, maps.Keys(telemetrytraces.CalculatedFields)...)
|
||||
|
||||
// Add matching intrinsic and matching calculated fields
|
||||
// These don't count towards the limit
|
||||
@@ -304,14 +324,14 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
|
||||
}
|
||||
|
||||
if found {
|
||||
if field, exists := tracestelemetryschema.IntrinsicFields[key]; exists {
|
||||
if field, exists := telemetrytraces.IntrinsicFields[key]; exists {
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if field, exists := tracestelemetryschema.CalculatedFields[key]; exists {
|
||||
if field, exists := telemetrytraces.CalculatedFields[key]; exists {
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
@@ -636,7 +656,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
complete := rowCount <= limit
|
||||
|
||||
staticKeys := []string{}
|
||||
staticKeys = append(staticKeys, maps.Keys(logstelemetryschema.IntrinsicFields)...)
|
||||
staticKeys = append(staticKeys, maps.Keys(telemetrylogs.IntrinsicFields)...)
|
||||
|
||||
// Add matching intrinsic and matching calculated fields
|
||||
// These don't count towards the limit
|
||||
@@ -650,7 +670,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
}
|
||||
|
||||
if found {
|
||||
if field, exists := logstelemetryschema.IntrinsicFields[key]; exists {
|
||||
if field, exists := telemetrylogs.IntrinsicFields[key]; exists {
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
@@ -878,7 +898,7 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
|
||||
complete := rowCount <= limit
|
||||
|
||||
// Add intrinsic audit fields (same as logs intrinsics: body, severity_text, etc.)
|
||||
staticKeys := maps.Keys(audittelemetryschema.IntrinsicFields)
|
||||
staticKeys := maps.Keys(telemetryaudit.IntrinsicFields)
|
||||
for _, key := range staticKeys {
|
||||
found := false
|
||||
for _, v := range searchTexts {
|
||||
@@ -889,7 +909,7 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
|
||||
}
|
||||
|
||||
if found {
|
||||
if field, exists := audittelemetryschema.IntrinsicFields[key]; exists {
|
||||
if field, exists := telemetryaudit.IntrinsicFields[key]; exists {
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
@@ -1156,7 +1176,7 @@ func enrichWithIntrinsicMetricKeys(keys map[string][]*telemetrytypes.TelemetryFi
|
||||
continue
|
||||
}
|
||||
|
||||
for name, key := range metricstelemetryschema.IntrinsicMetricFieldDefinitions {
|
||||
for name, key := range telemetrymetrics.IntrinsicMetricFieldDefinitions {
|
||||
if !selectorMatchesIntrinsicField(selector, key) {
|
||||
continue
|
||||
}
|
||||
@@ -1872,7 +1892,7 @@ func (t *telemetryMetaStore) getIntrinsicMetricFieldValues(ctx context.Context,
|
||||
instrumentationtypes.CodeNamespace: "metadata",
|
||||
instrumentationtypes.CodeFunctionName: "getIntrinsicMetricFieldValues",
|
||||
})
|
||||
key, ok := metricstelemetryschema.IntrinsicMetricFieldDefinitions[fieldValueSelector.Name]
|
||||
key, ok := telemetrymetrics.IntrinsicMetricFieldDefinitions[fieldValueSelector.Name]
|
||||
if !ok {
|
||||
return &telemetrytypes.TelemetryFieldValues{}, nil
|
||||
}
|
||||
@@ -1902,18 +1922,18 @@ func (t *telemetryMetaStore) getIntrinsicMetricFieldValues(ctx context.Context,
|
||||
tableName string
|
||||
extraConds []string
|
||||
}{
|
||||
{metricstelemetryschema.TimeseriesV41weekTableName, nil},
|
||||
{telemetrymetrics.TimeseriesV41weekTableName, nil},
|
||||
}
|
||||
if reductionEnabled {
|
||||
sources = append(sources,
|
||||
struct {
|
||||
tableName string
|
||||
extraConds []string
|
||||
}{metricstelemetryschema.TimeseriesV4BufferTableName, []string{"is_reduced = false"}},
|
||||
}{telemetrymetrics.TimeseriesV4BufferTableName, []string{"is_reduced = false"}},
|
||||
struct {
|
||||
tableName string
|
||||
extraConds []string
|
||||
}{metricstelemetryschema.TimeseriesV4ReducedTableName, nil},
|
||||
}{telemetrymetrics.TimeseriesV4ReducedTableName, nil},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -2242,7 +2262,7 @@ func (t *telemetryMetaStore) fetchMetricsTemporalityAndType(ctx context.Context,
|
||||
instrumentationtypes.CodeNamespace: "metadata",
|
||||
instrumentationtypes.CodeFunctionName: "fetchMetricsTemporalityAndType",
|
||||
})
|
||||
adjustedStartTs, adjustedEndTs, tsTableName, _ := metricstelemetryschema.WhichTSTableToUse(queryTimeRangeStartTs, queryTimeRangeEndTs, false, nil)
|
||||
adjustedStartTs, adjustedEndTs, tsTableName, _ := telemetrymetrics.WhichTSTableToUse(queryTimeRangeStartTs, queryTimeRangeEndTs, false, nil)
|
||||
|
||||
temporalities, types, err := t.fetchTemporalityTypeForTable(ctx, tsTableName, adjustedStartTs, adjustedEndTs, metricNames)
|
||||
if err != nil {
|
||||
@@ -2251,7 +2271,7 @@ func (t *telemetryMetaStore) fetchMetricsTemporalityAndType(ctx context.Context,
|
||||
|
||||
reduced := make(map[string]bool)
|
||||
if reductionEnabled {
|
||||
bufferTemporalities, bufferTypes, err := t.fetchTemporalityTypeForTable(ctx, metricstelemetryschema.TimeseriesV4BufferTableName, adjustedStartTs, adjustedEndTs, metricNames, "is_reduced = false")
|
||||
bufferTemporalities, bufferTypes, err := t.fetchTemporalityTypeForTable(ctx, telemetrymetrics.TimeseriesV4BufferTableName, adjustedStartTs, adjustedEndTs, metricNames, "is_reduced = false")
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
@@ -2264,7 +2284,7 @@ func (t *telemetryMetaStore) fetchMetricsTemporalityAndType(ctx context.Context,
|
||||
}
|
||||
}
|
||||
|
||||
reducedTemporalities, reducedTypes, err := t.fetchTemporalityTypeForTable(ctx, metricstelemetryschema.TimeseriesV4ReducedTableName, adjustedStartTs, adjustedEndTs, metricNames)
|
||||
reducedTemporalities, reducedTypes, err := t.fetchTemporalityTypeForTable(ctx, telemetrymetrics.TimeseriesV4ReducedTableName, adjustedStartTs, adjustedEndTs, metricNames)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
@@ -2596,13 +2616,13 @@ func (t *telemetryMetaStore) GetFirstSeenFromMetricMetadata(ctx context.Context,
|
||||
}
|
||||
|
||||
func (t *telemetryMetaStore) FetchLastSeenInfoMulti(ctx context.Context, orgID valuer.UUID, metricNames ...string) (map[string]int64, error) {
|
||||
lastSeenInfo, err := t.fetchLastSeenInfoForTable(ctx, metricstelemetryschema.TimeseriesV4TableName, metricNames)
|
||||
lastSeenInfo, err := t.fetchLastSeenInfoForTable(ctx, telemetrymetrics.TimeseriesV4TableName, metricNames)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) {
|
||||
reducedLastSeen, err := t.fetchLastSeenInfoForTable(ctx, metricstelemetryschema.TimeseriesV4ReducedTableName, metricNames)
|
||||
reducedLastSeen, err := t.fetchLastSeenInfoForTable(ctx, telemetrymetrics.TimeseriesV4ReducedTableName, metricNames)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -9,8 +9,13 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryaudit"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -37,6 +42,27 @@ func TestGetFirstSeenFromMetricMetadata(t *testing.T) {
|
||||
metadata := NewTelemetryMetaStore(
|
||||
instrumentationtest.New().ToProviderSettings(),
|
||||
mockTelemetryStore,
|
||||
telemetrytraces.DBName,
|
||||
telemetrytraces.TagAttributesV2TableName,
|
||||
telemetrytraces.SpanAttributesKeysTblName,
|
||||
telemetrytraces.SpanIndexV3TableName,
|
||||
telemetrymetrics.DBName,
|
||||
telemetrymetrics.AttributesMetadataTableName,
|
||||
telemetrymeter.DBName,
|
||||
telemetrymeter.SamplesAgg1dTableName,
|
||||
telemetrylogs.DBName,
|
||||
telemetrylogs.LogsV2TableName,
|
||||
telemetrylogs.TagAttributesV2TableName,
|
||||
telemetrylogs.LogAttributeKeysTblName,
|
||||
telemetrylogs.LogResourceKeysTblName,
|
||||
telemetryaudit.DBName,
|
||||
telemetryaudit.AuditLogsTableName,
|
||||
telemetryaudit.TagAttributesTableName,
|
||||
telemetryaudit.LogAttributeKeysTblName,
|
||||
telemetryaudit.LogResourceKeysTblName,
|
||||
DBName,
|
||||
AttributesMetadataTableName,
|
||||
ColumnEvolutionMetadataTableName,
|
||||
flaggertest.New(t),
|
||||
)
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package meterstatementbuilder
|
||||
package telemetrymeter
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -8,8 +8,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -22,7 +21,7 @@ type meterQueryStatementBuilder struct {
|
||||
metadataStore telemetrytypes.MetadataStore
|
||||
fm qbtypes.FieldMapper
|
||||
cb qbtypes.ConditionBuilder
|
||||
metricsStatementBuilder *metricsstatementbuilder.StatementBuilder
|
||||
metricsStatementBuilder *telemetrymetrics.MetricQueryStatementBuilder
|
||||
}
|
||||
|
||||
var _ qbtypes.StatementBuilder[qbtypes.MetricAggregation] = (*meterQueryStatementBuilder)(nil)
|
||||
@@ -32,9 +31,9 @@ func NewMeterQueryStatementBuilder(
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
fieldMapper qbtypes.FieldMapper,
|
||||
conditionBuilder qbtypes.ConditionBuilder,
|
||||
metricsStatementBuilder *metricsstatementbuilder.StatementBuilder,
|
||||
metricsStatementBuilder *telemetrymetrics.MetricQueryStatementBuilder,
|
||||
) *meterQueryStatementBuilder {
|
||||
metricsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema")
|
||||
metricsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrymeter")
|
||||
|
||||
return &meterQueryStatementBuilder{
|
||||
logger: metricsSettings.Logger(),
|
||||
@@ -54,7 +53,7 @@ func (b *meterQueryStatementBuilder) Build(
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
variables map[string]qbtypes.VariableItem,
|
||||
) (*qbtypes.Statement, error) {
|
||||
keySelectors := metricsstatementbuilder.GetKeySelectors(query)
|
||||
keySelectors := telemetrymetrics.GetKeySelectors(query)
|
||||
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, keySelectors)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -134,8 +133,8 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
sb.SelectMore(col)
|
||||
}
|
||||
|
||||
tbl := metertelemetryschema.WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
aggCol, err := metertelemetryschema.AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
tbl := WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
aggCol, err := AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -144,7 +143,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
}
|
||||
|
||||
sb.SelectMore(fmt.Sprintf("%s AS value", aggCol))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", metertelemetryschema.DBName, tbl))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", DBName, tbl))
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
sb.GTE("unix_milli", start),
|
||||
@@ -223,8 +222,8 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDelta(
|
||||
sb.SelectMore(col)
|
||||
}
|
||||
|
||||
tbl := metertelemetryschema.WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
aggCol, err := metertelemetryschema.AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality,
|
||||
tbl := WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
aggCol, err := AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality,
|
||||
query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
@@ -235,7 +234,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDelta(
|
||||
|
||||
sb.SelectMore(fmt.Sprintf("%s AS per_series_value", aggCol))
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", metertelemetryschema.DBName, tbl))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", DBName, tbl))
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
sb.GTE("unix_milli", start),
|
||||
@@ -301,14 +300,14 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
baseSb.SelectMore(col)
|
||||
}
|
||||
|
||||
tbl := metertelemetryschema.WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
aggCol, err := metertelemetryschema.AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
tbl := WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
aggCol, err := AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
baseSb.SelectMore(fmt.Sprintf("%s AS per_series_value", aggCol))
|
||||
|
||||
baseSb.From(fmt.Sprintf("%s.%s AS points", metertelemetryschema.DBName, tbl))
|
||||
baseSb.From(fmt.Sprintf("%s.%s AS points", DBName, tbl))
|
||||
baseSb.Where(
|
||||
baseSb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
baseSb.GTE("unix_milli", start),
|
||||
@@ -351,7 +350,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
for _, g := range query.GroupBy {
|
||||
wrapped.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
wrapped.SelectMore(fmt.Sprintf("%s AS per_series_value", metricsstatementbuilder.RateTmpl))
|
||||
wrapped.SelectMore(fmt.Sprintf("%s AS per_series_value", telemetrymetrics.RateTmpl))
|
||||
wrapped.From(fmt.Sprintf("(%s) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)", innerQuery))
|
||||
q, args := wrapped.BuildWithFlavor(sqlbuilder.ClickHouse, innerArgs...)
|
||||
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, nil
|
||||
@@ -362,7 +361,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
for _, g := range query.GroupBy {
|
||||
wrapped.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
wrapped.SelectMore(fmt.Sprintf("%s AS per_series_value", metricsstatementbuilder.IncreaseTmpl))
|
||||
wrapped.SelectMore(fmt.Sprintf("%s AS per_series_value", telemetrymetrics.IncreaseTmpl))
|
||||
wrapped.From(fmt.Sprintf("(%s) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)", innerQuery))
|
||||
q, args := wrapped.BuildWithFlavor(sqlbuilder.ClickHouse, innerArgs...)
|
||||
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, nil
|
||||
@@ -1,4 +1,4 @@
|
||||
package meterstatementbuilder
|
||||
package telemetrymeter
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -7,8 +7,7 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -159,8 +158,8 @@ func TestStatementBuilder(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
fm := metricstelemetryschema.NewFieldMapper()
|
||||
cb := metricstelemetryschema.NewConditionBuilder(fm)
|
||||
fm := telemetrymetrics.NewFieldMapper()
|
||||
cb := telemetrymetrics.NewConditionBuilder(fm)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
keys, err := telemetrytypestest.LoadFieldKeysFromJSON("testdata/keys_map.json")
|
||||
if err != nil {
|
||||
@@ -170,7 +169,7 @@ func TestStatementBuilder(t *testing.T) {
|
||||
|
||||
flagger := flaggertest.New(t)
|
||||
|
||||
metricStmtBuilder := metricsstatementbuilder.NewMetricQueryStatementBuilder(instrumentationtest.New().ToProviderSettings(), mockMetadataStore, fm, cb, flagger)
|
||||
metricStmtBuilder := telemetrymetrics.NewMetricQueryStatementBuilder(instrumentationtest.New().ToProviderSettings(), mockMetadataStore, fm, cb, flagger)
|
||||
|
||||
statementBuilder := NewMeterQueryStatementBuilder(
|
||||
instrumentationtest.New().ToProviderSettings(),
|
||||
@@ -1,4 +1,4 @@
|
||||
package metertelemetryschema
|
||||
package telemetrymeter
|
||||
|
||||
import (
|
||||
"time"
|
||||
@@ -1,4 +1,4 @@
|
||||
package metricstelemetryschema
|
||||
package telemetrymetrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package metricstelemetryschema
|
||||
package telemetrymetrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
37
pkg/telemetrymetrics/const.go
Normal file
37
pkg/telemetrymetrics/const.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package telemetrymetrics
|
||||
|
||||
import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
|
||||
var IntrinsicFields = []string{
|
||||
"temporality",
|
||||
"metric_name",
|
||||
"type",
|
||||
"is_monotonic",
|
||||
}
|
||||
|
||||
var IntrinsicMetricFieldDefinitions = map[string]telemetrytypes.TelemetryFieldKey{
|
||||
"metric_name": {
|
||||
Name: "metric_name",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextMetric,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
// "type": {
|
||||
// Name: "type",
|
||||
// Signal: telemetrytypes.SignalMetrics,
|
||||
// FieldContext: telemetrytypes.FieldContextMetric,
|
||||
// FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
// },
|
||||
// "temporality": {
|
||||
// Name: "temporality",
|
||||
// Signal: telemetrytypes.SignalMetrics,
|
||||
// FieldContext: telemetrytypes.FieldContextMetric,
|
||||
// FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
// },
|
||||
// "is_monotonic": {
|
||||
// Name: "is_monotonic",
|
||||
// Signal: telemetrytypes.SignalMetrics,
|
||||
// FieldContext: telemetrytypes.FieldContextMetric,
|
||||
// FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
// },
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package metricstelemetryschema
|
||||
package telemetrymetrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package metricstelemetryschema
|
||||
package telemetrymetrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package metricsstatementbuilder
|
||||
package telemetrymetrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -119,8 +118,8 @@ func TestReducedStatementBuilder(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
fm := metricstelemetryschema.NewFieldMapper()
|
||||
cb := metricstelemetryschema.NewConditionBuilder(fm)
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fl, err := flagger.New(context.Background(), instrumentationtest.New().ToProviderSettings(), flagger.Config{}, flagger.MustNewRegistry())
|
||||
require.NoError(t, err)
|
||||
sb := NewMetricQueryStatementBuilder(instrumentationtest.New().ToProviderSettings(), telemetrytypestest.NewMockMetadataStore(), fm, cb, fl)
|
||||
@@ -1,4 +1,4 @@
|
||||
package metricsstatementbuilder
|
||||
package telemetrymetrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -30,7 +29,7 @@ const (
|
||||
OthersMultiTemporality = `IF(LOWER(temporality) LIKE LOWER('delta'), %s, %s) AS per_series_value`
|
||||
)
|
||||
|
||||
type StatementBuilder struct {
|
||||
type MetricQueryStatementBuilder struct {
|
||||
logger *slog.Logger
|
||||
metadataStore telemetrytypes.MetadataStore
|
||||
fm qbtypes.FieldMapper
|
||||
@@ -38,7 +37,7 @@ type StatementBuilder struct {
|
||||
flagger flagger.Flagger
|
||||
}
|
||||
|
||||
var _ qbtypes.StatementBuilder[qbtypes.MetricAggregation] = (*StatementBuilder)(nil)
|
||||
var _ qbtypes.StatementBuilder[qbtypes.MetricAggregation] = (*MetricQueryStatementBuilder)(nil)
|
||||
|
||||
func NewMetricQueryStatementBuilder(
|
||||
settings factory.ProviderSettings,
|
||||
@@ -46,9 +45,9 @@ func NewMetricQueryStatementBuilder(
|
||||
fieldMapper qbtypes.FieldMapper,
|
||||
conditionBuilder qbtypes.ConditionBuilder,
|
||||
flagger flagger.Flagger,
|
||||
) *StatementBuilder {
|
||||
metricsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema")
|
||||
return &StatementBuilder{
|
||||
) *MetricQueryStatementBuilder {
|
||||
metricsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrymetrics")
|
||||
return &MetricQueryStatementBuilder{
|
||||
logger: metricsSettings.Logger(),
|
||||
metadataStore: metadataStore,
|
||||
fm: fieldMapper,
|
||||
@@ -90,7 +89,7 @@ func GetKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation])
|
||||
return keySelectors
|
||||
}
|
||||
|
||||
func (b *StatementBuilder) Build(
|
||||
func (b *MetricQueryStatementBuilder) Build(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
start uint64,
|
||||
@@ -127,7 +126,7 @@ func (b *StatementBuilder) Build(
|
||||
return b.buildPipelineStatement(ctx, orgID, start, end, query, keys, variables)
|
||||
}
|
||||
|
||||
func (b *StatementBuilder) buildPipelineStatement(
|
||||
func (b *MetricQueryStatementBuilder) buildPipelineStatement(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
start, end uint64,
|
||||
@@ -189,11 +188,11 @@ func (b *StatementBuilder) buildPipelineStatement(
|
||||
// shaped exactly like samples_v4 / time_series_v4, so once the table names are
|
||||
// chosen the rest of the pipeline is unchanged.
|
||||
useBuffer := agg.Reduced &&
|
||||
end-start < metricstelemetryschema.OneDayInMilliseconds &&
|
||||
start >= uint64(time.Now().UnixMilli())-metricstelemetryschema.OneDayInMilliseconds
|
||||
end-start < oneDayInMilliseconds &&
|
||||
start >= uint64(time.Now().UnixMilli())-oneDayInMilliseconds
|
||||
|
||||
samplesTable, _ := metricstelemetryschema.WhichSamplesTableToUse(start, end, agg.Type, agg.TimeAggregation, useBuffer, agg.TableHints)
|
||||
tsStart, tsEnd, _, tsTable := metricstelemetryschema.WhichTSTableToUse(start, end, useBuffer, agg.TableHints)
|
||||
samplesTable, _ := WhichSamplesTableToUse(start, end, agg.Type, agg.TimeAggregation, useBuffer, agg.TableHints)
|
||||
tsStart, tsEnd, _, tsTable := WhichTSTableToUse(start, end, useBuffer, agg.TableHints)
|
||||
|
||||
var timeSeriesCTE string
|
||||
var timeSeriesCTEArgs []any
|
||||
@@ -233,7 +232,7 @@ func (b *StatementBuilder) buildPipelineStatement(
|
||||
var tsCTE string
|
||||
var tsArgs []any
|
||||
// time series rows are written on hour boundaries
|
||||
tsStart := start - (start % metricstelemetryschema.OneHourInMilliseconds)
|
||||
tsStart := start - (start % oneHourInMilliseconds)
|
||||
if tsCTE, tsArgs, err = b.buildReducedTimeSeriesCTE(ctx, orgID, tsStart, end, query, keys, variables); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -283,7 +282,7 @@ func unionStatements(main, reduced *qbtypes.Statement, query qbtypes.QueryBuilde
|
||||
return &qbtypes.Statement{Query: q, Args: args, Warnings: warnings}, nil
|
||||
}
|
||||
|
||||
func (b *StatementBuilder) buildReducedTimeSeriesCTE(
|
||||
func (b *MetricQueryStatementBuilder) buildReducedTimeSeriesCTE(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
start, end uint64,
|
||||
@@ -313,7 +312,7 @@ func (b *StatementBuilder) buildReducedTimeSeriesCTE(
|
||||
}
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedLocalTableName))
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, TimeseriesV4ReducedLocalTableName))
|
||||
sb.Select("fingerprint")
|
||||
for _, g := range query.GroupBy {
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys)
|
||||
@@ -344,7 +343,7 @@ func (b *StatementBuilder) buildReducedTimeSeriesCTE(
|
||||
// spatial_aggregation_cte in one level with no per-series grouping, so shards
|
||||
// send one state per (step, group) instead of per (series, step, group).
|
||||
// FINAL still dedups recomputed 60s buckets at scan time.
|
||||
func (b *StatementBuilder) buildReducedSpatialAggFastPath(
|
||||
func (b *MetricQueryStatementBuilder) buildReducedSpatialAggFastPath(
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
timeSeriesCTE string,
|
||||
@@ -353,7 +352,7 @@ func (b *StatementBuilder) buildReducedSpatialAggFastPath(
|
||||
agg := query.Aggregations[0]
|
||||
stepSec := int64(query.StepInterval.Seconds())
|
||||
|
||||
value, _, ok := metricstelemetryschema.ReducedValueColumn(agg.Type, agg.SpaceAggregation)
|
||||
value, _, ok := ReducedValueColumn(agg.Type, agg.SpaceAggregation)
|
||||
if !ok {
|
||||
return "", nil, false
|
||||
}
|
||||
@@ -363,8 +362,8 @@ func (b *StatementBuilder) buildReducedSpatialAggFastPath(
|
||||
for _, g := range query.GroupBy {
|
||||
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
sb.SelectMore(fmt.Sprintf("%s AS value", metricstelemetryschema.ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec, value)))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points FINAL", metricstelemetryschema.DBName, metricstelemetryschema.WhichReducedSamplesTableToUse(agg.Type)))
|
||||
sb.SelectMore(fmt.Sprintf("%s AS value", ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec, value)))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points FINAL", DBName, WhichReducedSamplesTableToUse(agg.Type)))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.reduced_fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", agg.MetricName),
|
||||
@@ -378,7 +377,7 @@ func (b *StatementBuilder) buildReducedSpatialAggFastPath(
|
||||
return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args, true
|
||||
}
|
||||
|
||||
func (b *StatementBuilder) buildReducedTemporalAggregationCTE(
|
||||
func (b *MetricQueryStatementBuilder) buildReducedTemporalAggregationCTE(
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
timeSeriesCTE string,
|
||||
@@ -387,7 +386,7 @@ func (b *StatementBuilder) buildReducedTemporalAggregationCTE(
|
||||
agg := query.Aggregations[0]
|
||||
stepSec := int64(query.StepInterval.Seconds())
|
||||
|
||||
value, weight, ok := metricstelemetryschema.ReducedValueColumn(agg.Type, agg.SpaceAggregation)
|
||||
value, weight, ok := ReducedValueColumn(agg.Type, agg.SpaceAggregation)
|
||||
if !ok {
|
||||
return "", nil, false
|
||||
}
|
||||
@@ -400,13 +399,13 @@ func (b *StatementBuilder) buildReducedTemporalAggregationCTE(
|
||||
for _, g := range query.GroupBy {
|
||||
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
sb.SelectMore(fmt.Sprintf("%s AS per_series_value", metricstelemetryschema.ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec, value)))
|
||||
sb.SelectMore(fmt.Sprintf("%s AS per_series_value", ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec, value)))
|
||||
if weight != "" {
|
||||
// count_series is a series count, not additive over time, so the avg
|
||||
// denominator is reduced with avg
|
||||
sb.SelectMore(fmt.Sprintf("avg(%s) AS per_series_weight", weight))
|
||||
}
|
||||
sb.From(fmt.Sprintf("%s.%s AS points FINAL", metricstelemetryschema.DBName, metricstelemetryschema.WhichReducedSamplesTableToUse(agg.Type)))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points FINAL", DBName, WhichReducedSamplesTableToUse(agg.Type)))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.reduced_fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", agg.MetricName),
|
||||
@@ -420,7 +419,7 @@ func (b *StatementBuilder) buildReducedTemporalAggregationCTE(
|
||||
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, true
|
||||
}
|
||||
|
||||
func (b *StatementBuilder) buildReducedSpatialAggregationCTE(
|
||||
func (b *MetricQueryStatementBuilder) buildReducedSpatialAggregationCTE(
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
) (string, []any) {
|
||||
spatial := "sum(per_series_value)"
|
||||
@@ -447,7 +446,7 @@ func (b *StatementBuilder) buildReducedSpatialAggregationCTE(
|
||||
return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args
|
||||
}
|
||||
|
||||
func (b *StatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
func (b *MetricQueryStatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
samplesTable string,
|
||||
@@ -466,7 +465,7 @@ func (b *StatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
|
||||
aggCol, err := metricstelemetryschema.AggregationColumnForSamplesTable(
|
||||
aggCol, err := AggregationColumnForSamplesTable(
|
||||
samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation,
|
||||
)
|
||||
if err != nil {
|
||||
@@ -484,7 +483,7 @@ func (b *StatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
|
||||
sb.SelectMore(fmt.Sprintf("%s AS value", aggCol))
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", DBName, samplesTable))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
@@ -498,7 +497,7 @@ func (b *StatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args, nil
|
||||
}
|
||||
|
||||
func (b *StatementBuilder) buildTimeSeriesCTE(
|
||||
func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
start, end uint64,
|
||||
@@ -530,7 +529,7 @@ func (b *StatementBuilder) buildTimeSeriesCTE(
|
||||
}
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, tsTable))
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, tsTable))
|
||||
|
||||
sb.Select("fingerprint")
|
||||
for _, g := range query.GroupBy {
|
||||
@@ -553,7 +552,7 @@ func (b *StatementBuilder) buildTimeSeriesCTE(
|
||||
|
||||
// the buffer holds both raw rows and the reduced catalog rows; the raw read
|
||||
// only wants the original series
|
||||
if tsTable == metricstelemetryschema.TimeseriesV4BufferLocalTableName {
|
||||
if tsTable == TimeseriesV4BufferLocalTableName {
|
||||
sb.Where(sb.EQ("is_reduced", false))
|
||||
}
|
||||
|
||||
@@ -568,7 +567,7 @@ func (b *StatementBuilder) buildTimeSeriesCTE(
|
||||
return fmt.Sprintf("(%s) AS filtered_time_series", q), args, nil
|
||||
}
|
||||
|
||||
func (b *StatementBuilder) buildTemporalAggregationCTE(
|
||||
func (b *MetricQueryStatementBuilder) buildTemporalAggregationCTE(
|
||||
ctx context.Context,
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
@@ -585,7 +584,7 @@ func (b *StatementBuilder) buildTemporalAggregationCTE(
|
||||
return b.buildTemporalAggForMultipleTemporalities(ctx, start, end, query, samplesTable, timeSeriesCTE, timeSeriesCTEArgs)
|
||||
}
|
||||
|
||||
func (b *StatementBuilder) buildTemporalAggDelta(
|
||||
func (b *MetricQueryStatementBuilder) buildTemporalAggDelta(
|
||||
_ context.Context,
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
@@ -606,7 +605,7 @@ func (b *StatementBuilder) buildTemporalAggDelta(
|
||||
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
|
||||
aggCol, err := metricstelemetryschema.AggregationColumnForSamplesTable(samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation)
|
||||
aggCol, err := AggregationColumnForSamplesTable(samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -617,7 +616,7 @@ func (b *StatementBuilder) buildTemporalAggDelta(
|
||||
|
||||
sb.SelectMore(fmt.Sprintf("%s AS per_series_value", aggCol))
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", DBName, samplesTable))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
@@ -632,7 +631,7 @@ func (b *StatementBuilder) buildTemporalAggDelta(
|
||||
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, nil
|
||||
}
|
||||
|
||||
func (b *StatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
func (b *MetricQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
_ context.Context,
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
@@ -652,13 +651,13 @@ func (b *StatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
baseSb.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
|
||||
aggCol, err := metricstelemetryschema.AggregationColumnForSamplesTable(samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation)
|
||||
aggCol, err := AggregationColumnForSamplesTable(samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
baseSb.SelectMore(fmt.Sprintf("%s AS per_series_value", aggCol))
|
||||
|
||||
baseSb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
|
||||
baseSb.From(fmt.Sprintf("%s.%s AS points", DBName, samplesTable))
|
||||
baseSb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
baseSb.Where(
|
||||
baseSb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
@@ -698,7 +697,7 @@ func (b *StatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
}
|
||||
}
|
||||
|
||||
func (b *StatementBuilder) buildTemporalAggForMultipleTemporalities(
|
||||
func (b *MetricQueryStatementBuilder) buildTemporalAggForMultipleTemporalities(
|
||||
_ context.Context,
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
@@ -717,11 +716,11 @@ func (b *StatementBuilder) buildTemporalAggForMultipleTemporalities(
|
||||
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
|
||||
aggForDeltaTemporality, err := metricstelemetryschema.AggregationColumnForSamplesTable(samplesTable, metrictypes.Delta, query.Aggregations[0].TimeAggregation)
|
||||
aggForDeltaTemporality, err := AggregationColumnForSamplesTable(samplesTable, metrictypes.Delta, query.Aggregations[0].TimeAggregation)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
aggForCumulativeTemporality, err := metricstelemetryschema.AggregationColumnForSamplesTable(samplesTable, metrictypes.Cumulative, query.Aggregations[0].TimeAggregation)
|
||||
aggForCumulativeTemporality, err := AggregationColumnForSamplesTable(samplesTable, metrictypes.Cumulative, query.Aggregations[0].TimeAggregation)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -749,7 +748,7 @@ func (b *StatementBuilder) buildTemporalAggForMultipleTemporalities(
|
||||
sb.SelectMore(expr)
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", DBName, samplesTable))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
@@ -763,7 +762,7 @@ func (b *StatementBuilder) buildTemporalAggForMultipleTemporalities(
|
||||
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", queryWithWindowAndOrder), args, nil
|
||||
}
|
||||
|
||||
func (b *StatementBuilder) buildSpatialAggregationCTE(
|
||||
func (b *MetricQueryStatementBuilder) buildSpatialAggregationCTE(
|
||||
_ context.Context,
|
||||
_ uint64,
|
||||
_ uint64,
|
||||
@@ -789,7 +788,7 @@ func (b *StatementBuilder) buildSpatialAggregationCTE(
|
||||
return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args
|
||||
}
|
||||
|
||||
func (b *StatementBuilder) BuildFinalSelect(
|
||||
func (b *MetricQueryStatementBuilder) BuildFinalSelect(
|
||||
cteFragments []string,
|
||||
cteArgs [][]any,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
|
||||
@@ -834,7 +833,7 @@ func (b *StatementBuilder) BuildFinalSelect(
|
||||
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
|
||||
aggQuery, err := metricstelemetryschema.AggregationQueryForHistogramCountWithParams(query.Aggregations[0].ComparisonSpaceAggregationParam)
|
||||
aggQuery, err := AggregationQueryForHistogramCountWithParams(query.Aggregations[0].ComparisonSpaceAggregationParam)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package metricsstatementbuilder
|
||||
package telemetrymetrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -253,8 +252,8 @@ func TestStatementBuilder(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
fm := metricstelemetryschema.NewFieldMapper()
|
||||
cb := metricstelemetryschema.NewConditionBuilder(fm)
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
keys, err := telemetrytypestest.LoadFieldKeysFromJSON("testdata/keys_map.json")
|
||||
if err != nil {
|
||||
@@ -1,4 +1,4 @@
|
||||
package metricstelemetryschema
|
||||
package telemetrymetrics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -47,10 +46,10 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
OneHourInMilliseconds = uint64(time.Hour.Milliseconds() * 1)
|
||||
oneHourInMilliseconds = uint64(time.Hour.Milliseconds() * 1)
|
||||
sixHoursInMilliseconds = uint64(time.Hour.Milliseconds() * 6)
|
||||
OneDayInMilliseconds = uint64(time.Hour.Milliseconds() * 24)
|
||||
oneWeekInMilliseconds = uint64(OneDayInMilliseconds * 7)
|
||||
oneDayInMilliseconds = uint64(time.Hour.Milliseconds() * 24)
|
||||
oneWeekInMilliseconds = uint64(oneDayInMilliseconds * 7)
|
||||
|
||||
// when the query requests for almost 1 day, but not exactly 1 day, we need to add an offset to the end time
|
||||
// to make sure that we are using the correct table
|
||||
@@ -69,7 +68,7 @@ func WhichTSTableToUse(
|
||||
// the buffer holds the recent raw window for reduced metrics and has the same
|
||||
// shape as time_series_v4; round the start to the hour like the v4 table.
|
||||
if useBuffer {
|
||||
start = start - (start % (OneHourInMilliseconds))
|
||||
start = start - (start % (oneHourInMilliseconds))
|
||||
return start, end, TimeseriesV4BufferTableName, TimeseriesV4BufferLocalTableName
|
||||
}
|
||||
|
||||
@@ -81,7 +80,7 @@ func WhichTSTableToUse(
|
||||
switch tableHints.TimeSeriesTableName {
|
||||
case TimeseriesV4LocalTableName:
|
||||
// adjust the start time to nearest 1 hour
|
||||
start = start - (start % (OneHourInMilliseconds))
|
||||
start = start - (start % (oneHourInMilliseconds))
|
||||
distributedTableName = TimeseriesV4TableName
|
||||
case TimeseriesV46hrsLocalTableName:
|
||||
// adjust the start time to nearest 6 hours
|
||||
@@ -89,7 +88,7 @@ func WhichTSTableToUse(
|
||||
distributedTableName = TimeseriesV46hrsTableName
|
||||
case TimeseriesV41dayLocalTableName:
|
||||
// adjust the start time to nearest 1 day
|
||||
start = start - (start % (OneDayInMilliseconds))
|
||||
start = start - (start % (oneDayInMilliseconds))
|
||||
distributedTableName = TimeseriesV41dayTableName
|
||||
case TimeseriesV41weekLocalTableName:
|
||||
// adjust the start time to nearest 1 week
|
||||
@@ -108,17 +107,17 @@ func WhichTSTableToUse(
|
||||
var localTableName string
|
||||
if end-start < sixHoursInMilliseconds {
|
||||
// adjust the start time to nearest 1 hour
|
||||
start = start - (start % (OneHourInMilliseconds))
|
||||
start = start - (start % (oneHourInMilliseconds))
|
||||
distributedTableName = TimeseriesV4TableName
|
||||
localTableName = TimeseriesV4LocalTableName
|
||||
} else if end-start < OneDayInMilliseconds {
|
||||
} else if end-start < oneDayInMilliseconds {
|
||||
// adjust the start time to nearest 6 hours
|
||||
start = start - (start % (sixHoursInMilliseconds))
|
||||
distributedTableName = TimeseriesV46hrsTableName
|
||||
localTableName = TimeseriesV46hrsLocalTableName
|
||||
} else if end-start < oneWeekInMilliseconds {
|
||||
// adjust the start time to nearest 1 day
|
||||
start = start - (start % (OneDayInMilliseconds))
|
||||
start = start - (start % (oneDayInMilliseconds))
|
||||
distributedTableName = TimeseriesV41dayTableName
|
||||
localTableName = TimeseriesV41dayLocalTableName
|
||||
} else {
|
||||
@@ -207,7 +206,7 @@ func WhichSamplesTableToUse(
|
||||
return SamplesV4TableName, SamplesV4LocalTableName
|
||||
}
|
||||
|
||||
if end-start < OneDayInMilliseconds+offsetBucket {
|
||||
if end-start < oneDayInMilliseconds+offsetBucket {
|
||||
return SamplesV4TableName, SamplesV4LocalTableName
|
||||
} else if end-start < oneWeekInMilliseconds+offsetBucket {
|
||||
return SamplesV4Agg5mTableName, SamplesV4Agg5mLocalTableName
|
||||
@@ -431,37 +430,3 @@ func AggregationQueryForHistogramCountWithParams(param *metrictypes.ComparisonSp
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
var IntrinsicFields = []string{
|
||||
"temporality",
|
||||
"metric_name",
|
||||
"type",
|
||||
"is_monotonic",
|
||||
}
|
||||
|
||||
var IntrinsicMetricFieldDefinitions = map[string]telemetrytypes.TelemetryFieldKey{
|
||||
"metric_name": {
|
||||
Name: "metric_name",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextMetric,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
// "type": {
|
||||
// Name: "type",
|
||||
// Signal: telemetrytypes.SignalMetrics,
|
||||
// FieldContext: telemetrytypes.FieldContextMetric,
|
||||
// FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
// },
|
||||
// "temporality": {
|
||||
// Name: "temporality",
|
||||
// Signal: telemetrytypes.SignalMetrics,
|
||||
// FieldContext: telemetrytypes.FieldContextMetric,
|
||||
// FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
// },
|
||||
// "is_monotonic": {
|
||||
// Name: "is_monotonic",
|
||||
// Signal: telemetrytypes.SignalMetrics,
|
||||
// FieldContext: telemetrytypes.FieldContextMetric,
|
||||
// FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
// },
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package resourcefilter
|
||||
package telemetryresourcefilter
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package resourcefilter
|
||||
package telemetryresourcefilter
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package resourcefilter
|
||||
package telemetryresourcefilter
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package resourcefilter
|
||||
package telemetryresourcefilter
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,4 +1,4 @@
|
||||
package resourcefilter
|
||||
package telemetryresourcefilter
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -45,7 +45,7 @@ func New[T any](
|
||||
fullTextColumn *telemetrytypes.TelemetryFieldKey,
|
||||
fl flagger.Flagger,
|
||||
) *resourceFilterStatementBuilder[T] {
|
||||
set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/statementbuilder/resourcefilter")
|
||||
set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryresourcefilter")
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
return &resourceFilterStatementBuilder[T]{
|
||||
@@ -1,4 +1,4 @@
|
||||
package resourcefilter
|
||||
package telemetryresourcefilter
|
||||
|
||||
import (
|
||||
"context"
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user