Compare commits

..

3 Commits

Author SHA1 Message Date
srikanthccv
04be7222d7 feat(prometheus): add clickhouseprometheusv2 native read path
Second-generation ClickHouse-backed Prometheus provider: the stock engine
evaluates over a native storage.Querier instead of the v1 remote-read
adapter. Per-selector fetch windows, last-sample-per-step reduction for
subquery-free instant selectors (gated on prometheus.QueryTraits),
first-of-equal-timestamps dedup, identical-labelset merge, typed
invalid-input errors at the fetch ceilings instead of OOM.

Not wired: no factory registration, no config selection, nothing serves
from this package yet.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-29 12:09:15 +05:30
Srikanth Chekuri
bca2370862 test(promql): add upstream promqltest conformance corpus and suite (#12156)
Some checks are pending
build-staging / prepare (push) Waiting to run
build-staging / js-build (push) Blocked by required conditions
build-staging / go-build (push) Blocked by required conditions
build-staging / staging (push) Blocked by required conditions
Release Drafter / update_release_draft (push) Waiting to run
Freezes an absolute-truth oracle from prometheus@v0.311.3's own
promql/promqltest testdata: a generator command (scripts/promqltestcorpus, go run .)
evaluates upstream's load scripts with the vendored reference engine and
writes 755 cases to a committed corpus; a new integration suite replays
the ingestion and asserts /api/v5/query_range responses against it. The
known-divergences ledger is enforced exactly in both directions and is
empty: the serving path matches the reference engine on every case.

Also folds in the last serving-path fix the corpus surfaced: the v5
output filter dropped every __-prefixed label, mangling labelsets that
legitimately carry one (e.g. __address__); it now strips only known
storage keys (__temporality__, __scope./__resource. prefixes).

The metrics fixture writes registration rows per (series, hour bucket)
with hour-floored timestamps, matching the exporter; the queriermetrics
dormant-metric warning test now uses genuinely dormant data (production-
shaped registration shares an hour bucket with recently-stale data).

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-07-29 03:54:22 +00:00
Gaurav Tewari
591837152e fix(warning-popover): background color to l2 & message alignment (#12302)
Some checks failed
build-staging / prepare (push) Has been cancelled
Release Drafter / update_release_draft (push) Has been cancelled
build-staging / js-build (push) Has been cancelled
build-staging / go-build (push) Has been cancelled
build-staging / staging (push) Has been cancelled
* fix: update css styles

* feat: add warning popover

* chore: revert bg color change

* feat: update styles

* chore: css conventios

---------

Co-authored-by: Gaurav Tewari <tewarig@users.noreply.github.com>
2026-07-28 17:16:05 +00:00
131 changed files with 135093 additions and 1245 deletions

View File

@@ -54,6 +54,7 @@ jobs:
- querierscalar
- queriercommon
- rawexportdata
- promqlconformance
- querierauthz
- role
- rootuser

View File

@@ -2,9 +2,9 @@ package main
import (
"github.com/SigNoz/signoz/pkg/metercollector"
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/telemetrytraces"
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
"github.com/SigNoz/signoz/pkg/types/zeustypes"
)
@@ -25,8 +25,8 @@ var meterConfigs = []metercollector.Config{
Name: zeustypes.MeterLogSize,
Unit: zeustypes.MeterUnitBytes,
Aggregation: zeustypes.MeterAggregationSum,
DBName: logstelemetryschema.DBName,
TableName: logstelemetryschema.LogsV2LocalTableName,
DBName: telemetrylogs.DBName,
TableName: telemetrylogs.LogsV2LocalTableName,
DefaultRetentionDays: retentiontypes.DefaultLogsRetentionDays,
},
},
@@ -36,8 +36,8 @@ var meterConfigs = []metercollector.Config{
Name: zeustypes.MeterSpanSize,
Unit: zeustypes.MeterUnitBytes,
Aggregation: zeustypes.MeterAggregationSum,
DBName: tracestelemetryschema.DBName,
TableName: tracestelemetryschema.SpanIndexV3LocalTableName,
DBName: telemetrytraces.DBName,
TableName: telemetrytraces.SpanIndexV3LocalTableName,
DefaultRetentionDays: retentiontypes.DefaultTracesRetentionDays,
},
},
@@ -47,8 +47,8 @@ var meterConfigs = []metercollector.Config{
Name: zeustypes.MeterDatapointCount,
Unit: zeustypes.MeterUnitCount,
Aggregation: zeustypes.MeterAggregationSum,
DBName: metricstelemetryschema.DBName,
TableName: metricstelemetryschema.SamplesV4LocalTableName,
DBName: telemetrymetrics.DBName,
TableName: telemetrymetrics.SamplesV4LocalTableName,
DefaultRetentionDays: retentiontypes.DefaultMetricsRetentionDays,
},
},

View File

@@ -16,7 +16,7 @@ import (
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/metercollector"
"github.com/SigNoz/signoz/pkg/modules/retention"
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrymeter"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/licensetypes"
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
@@ -153,7 +153,7 @@ func (provider *Provider) Collect(
func buildOriginQuery(meterName string) (string, []any) {
sb := sqlbuilder.NewSelectBuilder()
sb.Select("toInt64(ifNull(min(unix_milli), 0))")
sb.From(metertelemetryschema.DBName + "." + metertelemetryschema.SamplesTableName)
sb.From(telemetrymeter.DBName + "." + telemetrymeter.SamplesTableName)
sb.Where(sb.Equal("metric_name", meterName))
return sb.BuildWithFlavor(sqlbuilder.ClickHouse)
}
@@ -171,7 +171,7 @@ func buildQuery(meterName string, segment *retentiontypes.RetentionPolicySegment
sb := sqlbuilder.NewSelectBuilder()
sb.Select(selects...)
sb.From(metertelemetryschema.DBName + "." + metertelemetryschema.SamplesTableName)
sb.From(telemetrymeter.DBName + "." + telemetrymeter.SamplesTableName)
sb.Where(
sb.Equal("metric_name", meterName),
sb.GTE("unix_milli", segment.StartMs),

View File

@@ -8,16 +8,16 @@ import (
sqlbuilder "github.com/huandu/go-sqlbuilder"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/metricreductionruletypes"
)
var (
reductionRulesTable = metricstelemetryschema.DBName + "." + metricstelemetryschema.ReductionRulesTableName
metadataTable = metricstelemetryschema.DBName + "." + metricstelemetryschema.AttributesMetadataTableName
bufferSeriesTable = metricstelemetryschema.DBName + "." + metricstelemetryschema.TimeseriesV4BufferTableName
reductionRulesTable = telemetrymetrics.DBName + "." + telemetrymetrics.ReductionRulesTableName
metadataTable = telemetrymetrics.DBName + "." + telemetrymetrics.AttributesMetadataTableName
bufferSeriesTable = telemetrymetrics.DBName + "." + telemetrymetrics.TimeseriesV4BufferTableName
)
const timeSeriesBucketMilli = int64(time.Hour / time.Millisecond)
@@ -192,7 +192,7 @@ func (c *clickhouse) ingestedSeriesCount(ctx context.Context, metricNames []stri
sb := sqlbuilder.NewSelectBuilder()
sb.Select("metric_name", "uniq(fingerprint)")
sb.From(metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4BufferTableName)
sb.From(telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName)
conds := []string{
sb.In("metric_name", names...),
sb.GE("unix_milli", startMs),
@@ -229,8 +229,8 @@ func (c *clickhouse) ingestedSeriesCount(ctx context.Context, metricNames []stri
// reduced sample tables.
func (c *clickhouse) reducedSeriesCount(ctx context.Context, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (map[string]uint64, error) {
out := make(map[string]uint64, len(metricNames))
for _, table := range []string{metricstelemetryschema.SamplesV4ReducedLastTableName, metricstelemetryschema.SamplesV4ReducedSumTableName} {
counts, err := c.reducedSeriesCountForTable(ctx, metricstelemetryschema.DBName+"."+table, metricNames, effectiveFrom, startMs, endMs)
for _, table := range []string{telemetrymetrics.SamplesV4ReducedLastTableName, telemetrymetrics.SamplesV4ReducedSumTableName} {
counts, err := c.reducedSeriesCountForTable(ctx, telemetrymetrics.DBName+"."+table, metricNames, effectiveFrom, startMs, endMs)
if err != nil {
return nil, err
}
@@ -300,9 +300,9 @@ func (c *clickhouse) RankByVolume(ctx context.Context, metricNames []string, eff
direction = "DESC"
}
ingestedTable := metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4BufferTableName
reducedLast := metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4ReducedLastTableName
reducedSum := metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4ReducedSumTableName
ingestedTable := telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName
reducedLast := telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4ReducedLastTableName
reducedSum := telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4ReducedSumTableName
sb := sqlbuilder.NewSelectBuilder()
sb.Select("base.metric_name AS metric_name", "ifNull(i.cnt, 0) AS ingested", "ifNull(d.cnt, 0) AS reduced")
@@ -352,16 +352,16 @@ func (c *clickhouse) SampleVolumeByMetric(ctx context.Context, metricNames []str
}
ctx = c.withThreads(ctx)
ingested, err := c.countSamplesByMetric(ctx, metricstelemetryschema.DBName+"."+metricstelemetryschema.SamplesV4BufferTableName, "count()", metricNames, effectiveFrom, startMs, endMs)
ingested, err := c.countSamplesByMetric(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4BufferTableName, "count()", metricNames, effectiveFrom, startMs, endMs)
if err != nil {
return nil, err
}
last, err := c.countSamplesByMetric(ctx, metricstelemetryschema.DBName+"."+metricstelemetryschema.SamplesV4ReducedLastTableName, "uniq(reduced_fingerprint, unix_milli)", metricNames, effectiveFrom, startMs, endMs)
last, err := c.countSamplesByMetric(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4ReducedLastTableName, "uniq(reduced_fingerprint, unix_milli)", metricNames, effectiveFrom, startMs, endMs)
if err != nil {
return nil, err
}
sum, err := c.countSamplesByMetric(ctx, metricstelemetryschema.DBName+"."+metricstelemetryschema.SamplesV4ReducedSumTableName, "uniq(reduced_fingerprint, unix_milli)", metricNames, effectiveFrom, startMs, endMs)
sum, err := c.countSamplesByMetric(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4ReducedSumTableName, "uniq(reduced_fingerprint, unix_milli)", metricNames, effectiveFrom, startMs, endMs)
if err != nil {
return nil, err
}
@@ -419,7 +419,7 @@ func (c *clickhouse) TotalVolume(ctx context.Context, startMs, endMs int64) (uin
sb := sqlbuilder.NewSelectBuilder()
sb.Select("uniq(fingerprint)", "count()")
sb.From(metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4BufferTableName)
sb.From(telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName)
sb.Where(sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs))
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -470,7 +470,7 @@ func (c *clickhouse) SampleTimeseries(ctx context.Context, ruledMetrics []string
func (c *clickhouse) totalSamplesByBucket(ctx context.Context, startMs, endMs int64) (map[int64]uint64, error) {
sb := sqlbuilder.NewSelectBuilder()
sb.Select(sampleBucketExpr, "count()")
sb.From(metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4BufferTableName)
sb.From(telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName)
sb.Where(sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs))
sb.GroupBy("bucket")
@@ -485,7 +485,7 @@ func (c *clickhouse) ruledIngestedSamplesByBucket(ctx context.Context, metricNam
sb := sqlbuilder.NewSelectBuilder()
sb.Select(sampleBucketExpr, "count()")
sb.From(metricstelemetryschema.DBName + "." + metricstelemetryschema.SamplesV4BufferTableName)
sb.From(telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName)
conds := []string{sb.In("metric_name", names...), sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs)}
if len(effectiveFrom) > 0 {
conds = append(conds, strictEffectiveFrom(sb, metricNames, effectiveFrom))
@@ -499,7 +499,7 @@ func (c *clickhouse) ruledIngestedSamplesByBucket(ctx context.Context, metricNam
// reduced 60s rows are versioned by computed_at, so count distinct buckets.
func (c *clickhouse) ruledRetainedSamplesByBucket(ctx context.Context, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (map[int64]uint64, error) {
out := make(map[int64]uint64)
for _, table := range []string{metricstelemetryschema.SamplesV4ReducedLastTableName, metricstelemetryschema.SamplesV4ReducedSumTableName} {
for _, table := range []string{telemetrymetrics.SamplesV4ReducedLastTableName, telemetrymetrics.SamplesV4ReducedSumTableName} {
names := make([]any, len(metricNames))
for i, name := range metricNames {
names[i] = name
@@ -507,7 +507,7 @@ func (c *clickhouse) ruledRetainedSamplesByBucket(ctx context.Context, metricNam
sb := sqlbuilder.NewSelectBuilder()
sb.Select(sampleBucketExpr, "uniq(reduced_fingerprint, unix_milli)")
sb.From(metricstelemetryschema.DBName + "." + table)
sb.From(telemetrymetrics.DBName + "." + table)
conds := []string{sb.In("metric_name", names...), sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs)}
if len(effectiveFrom) > 0 {
conds = append(conds, strictEffectiveFrom(sb, metricNames, effectiveFrom))

View File

@@ -1,6 +1,11 @@
.warning-popover-overlay {
--antd-arrow-background-color: var(--l2-background);
}
.warning-content {
display: flex;
flex-direction: column;
background-color: var(--l2-background);
// === SECTION: Summary (Top)
&__summary-section {
@@ -10,16 +15,25 @@
&__summary {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: var(--spacing-8);
padding: 16px;
}
&__summary-left {
display: flex;
align-items: baseline;
align-items: flex-start;
gap: 8px;
}
&__icon-wrapper {
display: flex;
align-items: center;
flex-shrink: 0;
height: var(--spacing-12);
}
&__summary-text {
display: flex;
flex-direction: column;

View File

@@ -2,6 +2,7 @@ import { ReactNode, useMemo } from 'react';
import { Color } from '@signozhq/design-tokens';
import { Button, Popover, PopoverProps } from 'antd';
import ErrorIcon from 'assets/Error';
import cx from 'classnames';
import OverlayScrollbar from 'components/OverlayScrollbar/OverlayScrollbar';
import { BookOpenText, ChevronsDown, TriangleAlert } from '@signozhq/icons';
import KeyValueLabel from 'periscope/components/KeyValueLabel';
@@ -30,16 +31,18 @@ export function WarningContent({ warning }: WarningContentProps): JSX.Element {
{/* Summary Header */}
<section className="warning-content__summary-section">
<header className="warning-content__summary">
<div className="warning-content__summary-left">
<div className="warning-content__icon-wrapper">
<ErrorIcon />
</div>
{(warningCode || warningMessage) && (
<div className="warning-content__summary-left">
<div className="warning-content__icon-wrapper">
<ErrorIcon />
</div>
<div className="warning-content__summary-text">
<h2 className="warning-content__warning-code">{warningCode}</h2>
<p className="warning-content__warning-message">{warningMessage}</p>
<div className="warning-content__summary-text">
<h2 className="warning-content__warning-code">{warningCode}</h2>
<p className="warning-content__warning-message">{warningMessage}</p>
</div>
</div>
</div>
)}
{warningUrl && (
<div className="warning-content__summary-right">
@@ -154,6 +157,10 @@ function WarningPopover({
overlayInnerStyle={{ padding: 0 }}
autoAdjustOverflow
{...popoverProps}
overlayClassName={cx(
'warning-popover-overlay',
popoverProps.overlayClassName,
)}
>
{children || (
<TriangleAlert

View File

@@ -5,14 +5,14 @@ import (
"fmt"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/valuer"
)
func (m *module) Collect(ctx context.Context, _ valuer.UUID) (map[string]any, error) {
stats := make(map[string]any)
metadataTable := fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName)
metadataTable := fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName)
var (
systemMetricCount uint64
k8sMetricCount uint64

View File

@@ -7,7 +7,7 @@ import (
"strings"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/valuer"
@@ -281,7 +281,7 @@ func (m *module) getPerGroupContainerStatusCounts(
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
// Built once; identical across the two fps CTEs (buildFilterClause hits the
// metadata store + parses the expression). AddWhereClause only reads it.
@@ -310,7 +310,7 @@ func (m *module) getPerGroupContainerStatusCounts(
)
}
stateFps.Select(stateFpsCols...)
stateFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
stateFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
stateFps.Where(
stateFps.E("metric_name", containerStatusStateMetricName),
stateFps.GE("unix_milli", tsAdjustedStart),
@@ -342,7 +342,7 @@ func (m *module) getPerGroupContainerStatusCounts(
containerState.Select(containerStateCols...)
containerState.From(fmt.Sprintf(
"%s.%s AS samples INNER JOIN state_fps AS fps ON samples.fingerprint = fps.fingerprint",
metricstelemetryschema.DBName, distributedSamplesTable,
telemetrymetrics.DBName, distributedSamplesTable,
))
containerState.Where(
containerState.E("samples.metric_name", containerStatusStateMetricName),
@@ -361,7 +361,7 @@ func (m *module) getPerGroupContainerStatusCounts(
fmt.Sprintf("JSONExtractString(labels, %s) AS container_name", reasonFps.Var(containerNameAttrKey)),
fmt.Sprintf("JSONExtractString(labels, %s) AS reason", reasonFps.Var(containerStatusReasonAttrKey)),
)
reasonFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
reasonFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
reasonFps.Where(
reasonFps.E("metric_name", containerStatusReasonMetricName),
reasonFps.GE("unix_milli", tsAdjustedStart),
@@ -391,7 +391,7 @@ func (m *module) getPerGroupContainerStatusCounts(
)
reasonInner.From(fmt.Sprintf(
"%s.%s AS samples INNER JOIN reason_fps AS fps ON samples.fingerprint = fps.fingerprint",
metricstelemetryschema.DBName, distributedSamplesTable,
telemetrymetrics.DBName, distributedSamplesTable,
))
reasonInner.Where(
reasonInner.E("samples.metric_name", containerStatusReasonMetricName),
@@ -546,7 +546,7 @@ func (m *module) getPerGroupContainerRestartCounts(
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
var (
filterClause *sqlbuilder.WhereClause
@@ -572,7 +572,7 @@ func (m *module) getPerGroupContainerRestartCounts(
)
}
restartFps.Select(restartFpsCols...)
restartFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
restartFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
restartFps.Where(
restartFps.E("metric_name", containerRestartsMetricName),
restartFps.GE("unix_milli", tsAdjustedStart),
@@ -602,7 +602,7 @@ func (m *module) getPerGroupContainerRestartCounts(
containerRestarts.Select(containerRestartsCols...)
containerRestarts.From(fmt.Sprintf(
"%s.%s AS samples INNER JOIN restart_fps AS fps ON samples.fingerprint = fps.fingerprint",
metricstelemetryschema.DBName, distributedSamplesTable,
telemetrymetrics.DBName, distributedSamplesTable,
))
containerRestarts.Where(
containerRestarts.E("samples.metric_name", containerRestartsMetricName),
@@ -690,7 +690,7 @@ func (m *module) getPerGroupContainerReadyCounts(
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
var (
filterClause *sqlbuilder.WhereClause
@@ -716,7 +716,7 @@ func (m *module) getPerGroupContainerReadyCounts(
)
}
readyFps.Select(readyFpsCols...)
readyFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
readyFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
readyFps.Where(
readyFps.E("metric_name", containerReadyMetricName),
readyFps.GE("unix_milli", tsAdjustedStart),
@@ -746,7 +746,7 @@ func (m *module) getPerGroupContainerReadyCounts(
containerReady.Select(containerReadyCols...)
containerReady.From(fmt.Sprintf(
"%s.%s AS samples INNER JOIN ready_fps AS fps ON samples.fingerprint = fps.fingerprint",
metricstelemetryschema.DBName, distributedSamplesTable,
telemetrymetrics.DBName, distributedSamplesTable,
))
containerReady.Where(
containerReady.E("samples.metric_name", containerReadyMetricName),

View File

@@ -9,7 +9,7 @@ import (
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/types/featuretypes"
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
@@ -344,11 +344,11 @@ func alignedMetricWindow(startMs, endMs int64) (
flooredEndMs = flooredEndMs - (flooredEndMs % (adjustStep * 1000))
}
tsAdjustedStartMs, _, distributedTSTable, localTSTable := metricstelemetryschema.WhichTSTableToUse(
tsAdjustedStartMs, _, distributedTSTable, localTSTable := telemetrymetrics.WhichTSTableToUse(
samplesAdjustedStartMs, flooredEndMs, false, nil,
)
distributedSamplesTable, localSamplesTable := metricstelemetryschema.WhichSamplesTableToUse(
distributedSamplesTable, localSamplesTable := telemetrymetrics.WhichSamplesTableToUse(
samplesAdjustedStartMs, flooredEndMs,
metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil,
)
@@ -362,7 +362,7 @@ func alignedMetricWindow(startMs, endMs int64) (
func (m *module) buildSamplesTblFingerprintSubQuery(metricNames []string, samplesTable string, flooredStart, flooredEnd uint64) *sqlbuilder.SelectBuilder {
fpSB := sqlbuilder.NewSelectBuilder()
fpSB.Select("DISTINCT fingerprint")
fpSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, samplesTable))
fpSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, samplesTable))
fpSB.Where(
fpSB.In("metric_name", sqlbuilder.List(metricNames)),
fpSB.GE("unix_milli", flooredStart),
@@ -376,7 +376,7 @@ func (m *module) buildSamplesTblFingerprintSubQuery(metricNames []string, sample
func (m *module) buildReducedSamplesTblFingerprintSubQuery(metricNames []string, flooredStart, flooredEnd uint64) *sqlbuilder.SelectBuilder {
lastSB := sqlbuilder.NewSelectBuilder()
lastSB.Select("reduced_fingerprint")
lastSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedLastTableName))
lastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName))
lastSB.Where(
lastSB.In("metric_name", sqlbuilder.List(metricNames)),
lastSB.GE("unix_milli", flooredStart),
@@ -385,7 +385,7 @@ func (m *module) buildReducedSamplesTblFingerprintSubQuery(metricNames []string,
sumSB := sqlbuilder.NewSelectBuilder()
sumSB.Select("reduced_fingerprint")
sumSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedSumTableName))
sumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName))
sumSB.Where(
sumSB.In("metric_name", sqlbuilder.List(metricNames)),
sumSB.GE("unix_milli", flooredStart),
@@ -478,7 +478,7 @@ func (m *module) getEarliestMetricTime(ctx context.Context, metricNames []string
sb := sqlbuilder.NewSelectBuilder()
sb.Select("min(first_reported_unix_milli) AS min_first_reported")
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
sb.Where(sb.In("metric_name", sqlbuilder.List(metricNames)))
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -504,7 +504,7 @@ func (m *module) getMetricsExistence(ctx context.Context, metricNames []string)
sb := sqlbuilder.NewSelectBuilder()
sb.Select("metric_name", "count(*) AS cnt")
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
sb.Where(sb.In("metric_name", sqlbuilder.List(metricNames)))
sb.GroupBy("metric_name")
@@ -549,7 +549,7 @@ func (m *module) getAttributesExistence(ctx context.Context, metricNames, attrNa
}
sb := sqlbuilder.NewSelectBuilder()
sb.Select("attr_name", "count(*) AS cnt")
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
sb.Where(
sb.In("metric_name", sqlbuilder.List(metricNames)),
sb.In("attr_name", sqlbuilder.List(attrNames)),
@@ -656,7 +656,7 @@ func (m *module) getMetadata(
rawSrc := sqlbuilder.NewSelectBuilder()
rawSrc.Select("labels", "unix_milli")
rawSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTableName))
rawSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTableName))
rawSrc.Where(
rawSrc.In("metric_name", sqlbuilder.List(metricNames)),
rawSrc.GE("unix_milli", tsAdjustedStartMs),
@@ -669,7 +669,7 @@ func (m *module) getMetadata(
reducedSrc := sqlbuilder.NewSelectBuilder()
reducedSrc.Select("labels", "unix_milli")
reducedSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
reducedSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
reducedSrc.Where(
reducedSrc.In("metric_name", sqlbuilder.List(metricNames)),
reducedSrc.GE("unix_milli", tsAdjustedStartMs),
@@ -683,7 +683,7 @@ func (m *module) getMetadata(
// Inner query reads over the union of raw + reduced series.
innerSB.From(innerSB.BuilderAs(sqlbuilder.UnionAll(rawSrc, reducedSrc), "series"))
} else {
innerSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTableName))
innerSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTableName))
innerSB.Where(
innerSB.In("metric_name", sqlbuilder.List(metricNames)),
innerSB.GE("unix_milli", tsAdjustedStartMs),
@@ -870,7 +870,7 @@ func (m *module) getPerGroupDistinctCounts(
rawSrc := sqlbuilder.NewSelectBuilder()
rawSrc.Select("labels")
rawSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTimeSeriesTbl))
rawSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTimeSeriesTbl))
rawSrc.Where(
rawSrc.In("metric_name", sqlbuilder.List(metricNames)),
rawSrc.GE("unix_milli", tsAdjustedStartMs),
@@ -883,7 +883,7 @@ func (m *module) getPerGroupDistinctCounts(
reducedSrc := sqlbuilder.NewSelectBuilder()
reducedSrc.Select("labels")
reducedSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
reducedSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
reducedSrc.Where(
reducedSrc.In("metric_name", sqlbuilder.List(metricNames)),
reducedSrc.GE("unix_milli", tsAdjustedStartMs),
@@ -896,7 +896,7 @@ func (m *module) getPerGroupDistinctCounts(
sb.From(sb.BuilderAs(sqlbuilder.UnionAll(rawSrc, reducedSrc), "series"))
} else {
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTimeSeriesTbl))
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTimeSeriesTbl))
sb.Where(
sb.In("metric_name", sqlbuilder.List(metricNames)),
sb.GE("unix_milli", tsAdjustedStartMs),

View File

@@ -7,7 +7,7 @@ import (
"strings"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/types/featuretypes"
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
@@ -72,7 +72,7 @@ func (m *module) getPerGroupHostStatusCounts(
rawSrc := sqlbuilder.NewSelectBuilder()
rawSrc.Select("labels")
rawSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTimeSeriesTableName))
rawSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTimeSeriesTableName))
rawSrc.Where(
rawSrc.In("metric_name", sqlbuilder.List(metricNames)),
rawSrc.GE("unix_milli", tsAdjustedStartMs),
@@ -85,7 +85,7 @@ func (m *module) getPerGroupHostStatusCounts(
reducedSrc := sqlbuilder.NewSelectBuilder()
reducedSrc.Select("labels")
reducedSrc.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
reducedSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
reducedSrc.Where(
reducedSrc.In("metric_name", sqlbuilder.List(metricNames)),
reducedSrc.GE("unix_milli", tsAdjustedStartMs),
@@ -101,7 +101,7 @@ func (m *module) getPerGroupHostStatusCounts(
fpSB := m.buildSamplesTblFingerprintSubQuery(metricNames, localSamplesTable, samplesStartMs, flooredEndMs)
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTimeSeriesTableName))
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTimeSeriesTableName))
sb.Where(
sb.In("metric_name", sqlbuilder.List(metricNames)),
sb.GE("unix_milli", tsAdjustedStartMs),
@@ -357,7 +357,7 @@ func (m *module) getActiveHostsQuery(metricNames []string, hostNameAttr string,
sb := sqlbuilder.NewSelectBuilder()
sb.Distinct()
sb.Select("attr_string_value")
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
sb.Where(
sb.In("metric_name", sqlbuilder.List(metricNames)),
sb.E("attr_name", hostNameAttr),

View File

@@ -9,7 +9,7 @@ import (
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/modules/inframonitoring"
"github.com/SigNoz/signoz/pkg/querier"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
@@ -40,8 +40,8 @@ func NewModule(
providerSettings factory.ProviderSettings,
cfg inframonitoring.Config,
) inframonitoring.Module {
fieldMapper := metricstelemetryschema.NewFieldMapper()
condBuilder := metricstelemetryschema.NewConditionBuilder(fieldMapper)
fieldMapper := telemetrymetrics.NewFieldMapper()
condBuilder := telemetrymetrics.NewConditionBuilder(fieldMapper)
return &module{
telemetryStore: telemetryStore,
telemetryMetadataStore: telemetryMetadataStore,

View File

@@ -7,7 +7,7 @@ import (
"strings"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/valuer"
@@ -186,7 +186,7 @@ func (m *module) getPerGroupNodeConditionCounts(
// Step-floor bounds + resolve tables in one shot to match QB v5 querier.
samplesStartMs, flooredEndMs, tsAdjustedStartMs, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
// ----- timeSeriesFPs -----
timeSeriesFPs := sqlbuilder.NewSelectBuilder()
@@ -200,7 +200,7 @@ func (m *module) getPerGroupNodeConditionCounts(
)
}
timeSeriesFPs.Select(timeSeriesFPsSelectCols...)
timeSeriesFPs.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
timeSeriesFPs.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
timeSeriesFPs.Where(
timeSeriesFPs.E("metric_name", nodeConditionMetricName),
timeSeriesFPs.GE("unix_milli", tsAdjustedStartMs),
@@ -237,7 +237,7 @@ func (m *module) getPerGroupNodeConditionCounts(
latestConditionPerNode.Select(latestConditionPerNodeSelectCols...)
latestConditionPerNode.From(fmt.Sprintf(
"%s.%s AS samples INNER JOIN time_series_fps AS tsfp ON samples.fingerprint = tsfp.fingerprint",
metricstelemetryschema.DBName, distributedSamplesTable,
telemetrymetrics.DBName, distributedSamplesTable,
))
latestConditionPerNode.Where(
latestConditionPerNode.E("samples.metric_name", nodeConditionMetricName),

View File

@@ -8,7 +8,7 @@ import (
"time"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/valuer"
@@ -284,7 +284,7 @@ func (m *module) getPerGroupPodStatusCounts(
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
// Build the merged filter clause once; it's identical across the three fps
// CTEs, and buildFilterClause hits the metadata store + parses the
@@ -313,7 +313,7 @@ func (m *module) getPerGroupPodStatusCounts(
)
}
phaseFps.Select(phaseFpsCols...)
phaseFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
phaseFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
phaseFps.Where(
phaseFps.E("metric_name", podPhaseMetricName),
phaseFps.GE("unix_milli", tsAdjustedStart),
@@ -340,7 +340,7 @@ func (m *module) getPerGroupPodStatusCounts(
phasePerPod.Select(phasePerPodCols...)
phasePerPod.From(fmt.Sprintf(
"%s.%s AS samples INNER JOIN phase_fps AS fps ON samples.fingerprint = fps.fingerprint",
metricstelemetryschema.DBName, distributedSamplesTable,
telemetrymetrics.DBName, distributedSamplesTable,
))
phasePerPod.Where(
phasePerPod.E("samples.metric_name", podPhaseMetricName),
@@ -357,7 +357,7 @@ func (m *module) getPerGroupPodStatusCounts(
"fingerprint",
fmt.Sprintf("JSONExtractString(labels, %s) AS pod_uid", podReasonFps.Var(podUIDAttrKey)),
)
podReasonFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
podReasonFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
podReasonFps.Where(
podReasonFps.E("metric_name", podStatusReasonMetricName),
podReasonFps.GE("unix_milli", tsAdjustedStart),
@@ -377,7 +377,7 @@ func (m *module) getPerGroupPodStatusCounts(
)
podReasonPerPod.From(fmt.Sprintf(
"%s.%s AS samples INNER JOIN pod_reason_fps AS fps ON samples.fingerprint = fps.fingerprint",
metricstelemetryschema.DBName, distributedSamplesTable,
telemetrymetrics.DBName, distributedSamplesTable,
))
podReasonPerPod.Where(
podReasonPerPod.E("samples.metric_name", podStatusReasonMetricName),
@@ -396,7 +396,7 @@ func (m *module) getPerGroupPodStatusCounts(
fmt.Sprintf("JSONExtractString(labels, %s) AS container_name", containerReasonFps.Var(containerNameAttrKey)),
fmt.Sprintf("JSONExtractString(labels, %s) AS reason", containerReasonFps.Var(containerStatusReasonAttrKey)),
)
containerReasonFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
containerReasonFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
containerReasonFps.Where(
containerReasonFps.E("metric_name", containerStatusReasonMetricName),
containerReasonFps.GE("unix_milli", tsAdjustedStart),
@@ -432,7 +432,7 @@ func (m *module) getPerGroupPodStatusCounts(
)
containerInner.From(fmt.Sprintf(
"%s.%s AS samples INNER JOIN container_reason_fps AS fps ON samples.fingerprint = fps.fingerprint",
metricstelemetryschema.DBName, distributedSamplesTable,
telemetrymetrics.DBName, distributedSamplesTable,
))
containerInner.Where(
containerInner.E("samples.metric_name", containerStatusReasonMetricName),
@@ -611,7 +611,7 @@ func (m *module) getPerGroupPodRestartCounts(
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
valueCol := metricstelemetryschema.ValueColumnForSamplesTable(distributedSamplesTable)
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
var (
filterClause *sqlbuilder.WhereClause
@@ -637,7 +637,7 @@ func (m *module) getPerGroupPodRestartCounts(
)
}
restartFps.Select(restartFpsCols...)
restartFps.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTimeSeriesTable))
restartFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
restartFps.Where(
restartFps.E("metric_name", containerRestartsMetricName),
restartFps.GE("unix_milli", tsAdjustedStart),
@@ -667,7 +667,7 @@ func (m *module) getPerGroupPodRestartCounts(
containerRestarts.Select(containerRestartsCols...)
containerRestarts.From(fmt.Sprintf(
"%s.%s AS samples INNER JOIN restart_fps AS fps ON samples.fingerprint = fps.fingerprint",
metricstelemetryschema.DBName, distributedSamplesTable,
telemetrymetrics.DBName, distributedSamplesTable,
))
containerRestarts.Where(
containerRestarts.E("samples.metric_name", containerRestartsMetricName),

View File

@@ -19,8 +19,8 @@ import (
"github.com/SigNoz/signoz/pkg/modules/dashboard"
"github.com/SigNoz/signoz/pkg/modules/metricsexplorer"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrymeter"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
@@ -49,8 +49,8 @@ type module struct {
// NewModule constructs the metrics module with the provided dependencies.
func NewModule(ts telemetrystore.TelemetryStore, telemetryMetadataStore telemetrytypes.MetadataStore, cache cache.Cache, ruleStore ruletypes.RuleStore, dashboardModule dashboard.Module, fl flagger.Flagger, providerSettings factory.ProviderSettings, cfg metricsexplorer.Config) metricsexplorer.Module {
fieldMapper := metricstelemetryschema.NewFieldMapper()
condBuilder := metricstelemetryschema.NewConditionBuilder(fieldMapper)
fieldMapper := telemetrymetrics.NewFieldMapper()
condBuilder := telemetrymetrics.NewConditionBuilder(fieldMapper)
return &module{
telemetryStore: ts,
fieldMapper: fieldMapper,
@@ -89,7 +89,7 @@ func (m *module) listMeterMetrics(ctx context.Context, params *metricsexplorerty
"argMax(temporality, unix_milli) AS temporality",
"argMax(is_monotonic, unix_milli) AS is_monotonic",
)
sb.From(fmt.Sprintf("%s.%s", metertelemetryschema.DBName, metertelemetryschema.SamplesTableName))
sb.From(fmt.Sprintf("%s.%s", telemetrymeter.DBName, telemetrymeter.SamplesTableName))
if params.Start != nil && params.End != nil {
sb.Where(sb.Between("unix_milli", *params.Start, *params.End))
@@ -160,11 +160,11 @@ func (m *module) listMetrics(ctx context.Context, orgID valuer.UUID, params *met
hasRange := params.Start != nil && params.End != nil
if hasRange {
var distributedTsTable string
rangeStart, rangeEnd, distributedTsTable, _ = metricstelemetryschema.WhichTSTableToUse(uint64(*params.Start), uint64(*params.End), false, nil)
rawSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTsTable))
rangeStart, rangeEnd, distributedTsTable, _ = telemetrymetrics.WhichTSTableToUse(uint64(*params.Start), uint64(*params.End), false, nil)
rawSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
rawSB.Where(rawSB.Between("unix_milli", rangeStart, rangeEnd))
} else {
rawSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV41weekTableName))
rawSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV41weekTableName))
}
applyListFilters(rawSB)
@@ -174,7 +174,7 @@ func (m *module) listMetrics(ctx context.Context, orgID valuer.UUID, params *met
if reductionEnabled {
reducedSB := sqlbuilder.NewSelectBuilder()
reducedSB.Select("DISTINCT metric_name")
reducedSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
reducedSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
if hasRange {
reducedSB.Where(reducedSB.Between("unix_milli", rangeStart, rangeEnd))
}
@@ -512,7 +512,7 @@ func (m *module) CheckMetricExists(ctx context.Context, orgID valuer.UUID, metri
sb := sqlbuilder.NewSelectBuilder()
sb.Select("count(*) > 0 as metricExists")
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
sb.Where(sb.E("metric_name", metricName))
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -534,7 +534,7 @@ func (m *module) HasNonSigNozMetrics(ctx context.Context) (bool, error) {
sb := sqlbuilder.NewSelectBuilder()
sb.Select("count(*) > 0")
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV41weekTableName))
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV41weekTableName))
sb.Where("metric_name NOT LIKE 'signoz_%'")
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -570,10 +570,10 @@ func (m *module) InspectMetrics(
return nil, err
}
tsStart, _, tsTable, _ := metricstelemetryschema.WhichTSTableToUse(start, end, false, nil)
tsStart, _, tsTable, _ := telemetrymetrics.WhichTSTableToUse(start, end, false, nil)
tsSb := sqlbuilder.NewSelectBuilder()
tsSb.Select("fingerprint", "labels")
tsSb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, tsTable))
tsSb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, tsTable))
tsSb.Where(
tsSb.E("metric_name", req.MetricName),
tsSb.GE("unix_milli", tsStart),
@@ -625,7 +625,7 @@ func (m *module) InspectMetrics(
samplesSb := sqlbuilder.NewSelectBuilder()
samplesSb.Select("fingerprint", "unix_milli", "value")
samplesSb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4TableName))
samplesSb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4TableName))
samplesSb.Where(
samplesSb.In("fingerprint", fingerprints...),
samplesSb.E("metric_name", req.MetricName),
@@ -717,7 +717,7 @@ func (m *module) fetchUpdatedMetadata(ctx context.Context, orgID valuer.UUID, me
"argMax(temporality, created_at) AS temporality",
"argMax(is_monotonic, created_at) AS is_monotonic",
)
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.UpdatedMetadataTableName))
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.UpdatedMetadataTableName))
sb.Where(sb.In("metric_name", args...))
sb.GroupBy("metric_name")
@@ -777,7 +777,7 @@ func (m *module) fetchTimeseriesMetadata(ctx context.Context, orgID valuer.UUID,
"anyLast(temporality) AS temporality",
"argMax(is_monotonic, unix_milli) AS is_monotonic",
)
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4TableName))
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4TableName))
sb.Where(sb.In("metric_name", args...))
sb.GroupBy("metric_name")
@@ -891,7 +891,7 @@ func (m *module) checkForLabelInMetric(ctx context.Context, metricName string, l
sb := sqlbuilder.NewSelectBuilder()
sb.Select("count(*) > 0 AS has_label")
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
sb.Where(sb.E("metric_name", metricName))
sb.Where(sb.E("attr_name", label))
sb.Limit(1)
@@ -914,7 +914,7 @@ func (m *module) insertMetricsMetadata(ctx context.Context, orgID valuer.UUID, r
createdAt := time.Now().UnixMilli()
ib := sqlbuilder.NewInsertBuilder()
ib.InsertInto(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.UpdatedMetadataTableName))
ib.InsertInto(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.UpdatedMetadataTableName))
ib.Cols("metric_name", "temporality", "is_monotonic", "type", "description", "unit", "created_at")
ib.Values(
req.MetricName,
@@ -1016,9 +1016,9 @@ func (m *module) fetchMetricsStatsWithSamples(
}
}
start, end, distributedTsTable, localTsTable := metricstelemetryschema.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
distributedSamplesTable, _ := metricstelemetryschema.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil)
countExp := metricstelemetryschema.CountExpressionForSamplesTable(distributedSamplesTable)
start, end, distributedTsTable, localTsTable := telemetrymetrics.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
distributedSamplesTable, _ := telemetrymetrics.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil)
countExp := telemetrymetrics.CountExpressionForSamplesTable(distributedSamplesTable)
// Timeseries counts per metric
tsSB := sqlbuilder.NewSelectBuilder()
@@ -1026,7 +1026,7 @@ func (m *module) fetchMetricsStatsWithSamples(
"metric_name",
"uniq(fingerprint) AS timeseries",
)
tsSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTsTable))
tsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
tsSB.Where(tsSB.Between("unix_milli", start, end))
tsSB.Where("NOT startsWith(metric_name, 'signoz')")
if filterWhereClause != nil {
@@ -1040,7 +1040,7 @@ func (m *module) fetchMetricsStatsWithSamples(
"metric_name",
fmt.Sprintf("%s AS samples", countExp),
)
samplesSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedSamplesTable))
samplesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedSamplesTable))
samplesSB.Where(samplesSB.Between("unix_milli", req.Start, req.End))
samplesSB.Where("NOT startsWith(metric_name, 'signoz')")
@@ -1053,7 +1053,7 @@ func (m *module) fetchMetricsStatsWithSamples(
if filterWhereClause != nil {
fingerprintSB := sqlbuilder.NewSelectBuilder()
fingerprintSB.Select("fingerprint")
fingerprintSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTsTable))
fingerprintSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTsTable))
fingerprintSB.Where(fingerprintSB.Between("unix_milli", start, end))
fingerprintSB.Where("NOT startsWith(metric_name, 'signoz')")
fingerprintSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause))
@@ -1064,7 +1064,7 @@ func (m *module) fetchMetricsStatsWithSamples(
} else {
metricNamesSB := sqlbuilder.NewSelectBuilder()
metricNamesSB.Select("DISTINCT metric_name")
metricNamesSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTsTable))
metricNamesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTsTable))
metricNamesSB.Where(metricNamesSB.Between("unix_milli", start, end))
metricNamesSB.Where("NOT startsWith(metric_name, 'signoz')")
@@ -1090,18 +1090,18 @@ func (m *module) fetchMetricsStatsWithSamples(
// the data lands either of the _reduced_last/_reduced_sum tables
reducedLastSB := sqlbuilder.NewSelectBuilder()
reducedLastSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt")
reducedLastSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedLastTableName))
reducedLastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName))
reducedLastSB.Where(reducedLastSB.Between("unix_milli", req.Start, req.End))
reducedLastSB.Where("NOT startsWith(metric_name, 'signoz')")
reducedSumSB := sqlbuilder.NewSelectBuilder()
reducedSumSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt")
reducedSumSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedSumTableName))
reducedSumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName))
reducedSumSB.Where(reducedSumSB.Between("unix_milli", req.Start, req.End))
reducedSumSB.Where("NOT startsWith(metric_name, 'signoz')")
// separate query for reduced series counts
reducedTsSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
reducedTsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
reducedTsSB.Where(reducedTsSB.Between("unix_milli", start, end))
reducedTsSB.Where("NOT startsWith(metric_name, 'signoz')")
reducedTsSB.GroupBy("metric_name")
@@ -1112,7 +1112,7 @@ func (m *module) fetchMetricsStatsWithSamples(
// samples uses a separate cte with local table
reducedFpSB := sqlbuilder.NewSelectBuilder()
reducedFpSB.Select("fingerprint")
reducedFpSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedLocalTableName))
reducedFpSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedLocalTableName))
reducedFpSB.Where(reducedFpSB.Between("unix_milli", start, end))
reducedFpSB.Where("NOT startsWith(metric_name, 'signoz')")
reducedFpSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause))
@@ -1220,11 +1220,11 @@ func (m *module) computeTimeseriesTreemap(ctx context.Context, orgID valuer.UUID
}
}
start, end, distributedTsTable, _ := metricstelemetryschema.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
start, end, distributedTsTable, _ := telemetrymetrics.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
totalTSBuilder := sqlbuilder.NewSelectBuilder()
totalTSBuilder.Select("uniq(fingerprint) AS total_time_series")
totalTSBuilder.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTsTable))
totalTSBuilder.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
totalTSBuilder.Where(totalTSBuilder.Between("unix_milli", start, end))
metricsSB := sqlbuilder.NewSelectBuilder()
@@ -1232,7 +1232,7 @@ func (m *module) computeTimeseriesTreemap(ctx context.Context, orgID valuer.UUID
"metric_name",
"uniq(fingerprint) AS total_value",
)
metricsSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTsTable))
metricsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
metricsSB.Where(metricsSB.Between("unix_milli", start, end))
metricsSB.Where("NOT startsWith(metric_name, 'signoz')")
if filterWhereClause != nil {
@@ -1244,7 +1244,7 @@ func (m *module) computeTimeseriesTreemap(ctx context.Context, orgID valuer.UUID
if reductionEnabled {
reducedTotalTSBuilder := sqlbuilder.NewSelectBuilder()
reducedTotalTSBuilder.Select("uniq(fingerprint) AS total_time_series")
reducedTotalTSBuilder.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
reducedTotalTSBuilder.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
reducedTotalTSBuilder.Where(reducedTotalTSBuilder.Between("unix_milli", start, end))
reducedMetricsSB := sqlbuilder.NewSelectBuilder()
@@ -1252,7 +1252,7 @@ func (m *module) computeTimeseriesTreemap(ctx context.Context, orgID valuer.UUID
"metric_name",
"uniq(fingerprint) AS total_value",
)
reducedMetricsSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
reducedMetricsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
reducedMetricsSB.Where(reducedMetricsSB.Between("unix_milli", start, end))
reducedMetricsSB.Where("NOT startsWith(metric_name, 'signoz')")
if filterWhereClause != nil {
@@ -1338,15 +1338,15 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
}
}
start, end, distributedTsTable, localTsTable := metricstelemetryschema.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
distributedSamplesTable, _ := metricstelemetryschema.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil)
countExp := metricstelemetryschema.CountExpressionForSamplesTable(distributedSamplesTable)
start, end, distributedTsTable, localTsTable := telemetrymetrics.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
distributedSamplesTable, _ := telemetrymetrics.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil)
countExp := telemetrymetrics.CountExpressionForSamplesTable(distributedSamplesTable)
candidateLimit := req.Limit + 50
metricCandidatesSB := sqlbuilder.NewSelectBuilder()
metricCandidatesSB.Select("metric_name")
metricCandidatesSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedTsTable))
metricCandidatesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
metricCandidatesSB.Where("NOT startsWith(metric_name, 'signoz')")
metricCandidatesSB.Where(metricCandidatesSB.Between("unix_milli", start, end))
if filterWhereClause != nil {
@@ -1364,7 +1364,7 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
if reductionEnabled {
reducedCandidatesSB = sqlbuilder.NewSelectBuilder()
reducedCandidatesSB.Select("metric_name")
reducedCandidatesSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedTableName))
reducedCandidatesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
reducedCandidatesSB.Where("NOT startsWith(metric_name, 'signoz')")
reducedCandidatesSB.Where(reducedCandidatesSB.Between("unix_milli", start, end))
if filterWhereClause != nil {
@@ -1379,7 +1379,7 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
totalSamplesSB := sqlbuilder.NewSelectBuilder()
totalSamplesSB.Select(fmt.Sprintf("%s AS total_samples", countExp))
totalSamplesSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedSamplesTable))
totalSamplesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedSamplesTable))
totalSamplesSB.Where(totalSamplesSB.Between("unix_milli", req.Start, req.End))
sampleCountsSB := sqlbuilder.NewSelectBuilder()
@@ -1387,7 +1387,7 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
"metric_name",
fmt.Sprintf("%s AS samples", countExp),
)
sampleCountsSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, distributedSamplesTable))
sampleCountsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedSamplesTable))
sampleCountsSB.Where(sampleCountsSB.Between("unix_milli", req.Start, req.End))
sampleCountsSB.Where("metric_name GLOBAL IN (SELECT metric_name FROM __metric_candidates)")
@@ -1397,13 +1397,13 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
if reductionEnabled {
reducedLastSB = sqlbuilder.NewSelectBuilder()
reducedLastSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt")
reducedLastSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedLastTableName))
reducedLastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName))
reducedLastSB.Where(reducedLastSB.Between("unix_milli", req.Start, req.End))
reducedLastSB.Where("metric_name GLOBAL IN (SELECT metric_name FROM __reduced_metric_candidates)")
reducedSumSB = sqlbuilder.NewSelectBuilder()
reducedSumSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt")
reducedSumSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedSumTableName))
reducedSumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName))
reducedSumSB.Where(reducedSumSB.Between("unix_milli", req.Start, req.End))
reducedSumSB.Where("metric_name GLOBAL IN (SELECT metric_name FROM __reduced_metric_candidates)")
}
@@ -1411,7 +1411,7 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
if filterWhereClause != nil {
fingerprintSB := sqlbuilder.NewSelectBuilder()
fingerprintSB.Select("fingerprint")
fingerprintSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, localTsTable))
fingerprintSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTsTable))
fingerprintSB.Where(fingerprintSB.Between("unix_milli", start, end))
fingerprintSB.Where("NOT startsWith(metric_name, 'signoz')")
fingerprintSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause))
@@ -1423,7 +1423,7 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
if reductionEnabled {
reducedFingerprintSB := sqlbuilder.NewSelectBuilder()
reducedFingerprintSB.Select("fingerprint")
reducedFingerprintSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedLocalTableName))
reducedFingerprintSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedLocalTableName))
reducedFingerprintSB.Where(reducedFingerprintSB.Between("unix_milli", start, end))
reducedFingerprintSB.Where("NOT startsWith(metric_name, 'signoz')")
reducedFingerprintSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause))
@@ -1457,12 +1457,12 @@ func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, r
// Grand total includes reduced sample volume so percentages stay consistent.
reducedTotalLastSB := sqlbuilder.NewSelectBuilder()
reducedTotalLastSB.Select("uniq(reduced_fingerprint, unix_milli) AS cnt")
reducedTotalLastSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedLastTableName))
reducedTotalLastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName))
reducedTotalLastSB.Where(reducedTotalLastSB.Between("unix_milli", req.Start, req.End))
reducedTotalSumSB := sqlbuilder.NewSelectBuilder()
reducedTotalSumSB.Select("uniq(reduced_fingerprint, unix_milli) AS cnt")
reducedTotalSumSB.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4ReducedSumTableName))
reducedTotalSumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName))
reducedTotalSumSB.Where(reducedTotalSumSB.Between("unix_milli", req.Start, req.End))
reducedTotalSamplesSB := sqlbuilder.NewSelectBuilder()
@@ -1552,7 +1552,7 @@ func (m *module) getMetricDataPoints(ctx context.Context, metricName string) (ui
sb := sqlbuilder.NewSelectBuilder()
sb.Select("sum(count) AS data_points")
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4Agg30mTableName))
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4Agg30mTableName))
sb.Where(sb.E("metric_name", metricName))
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -1574,7 +1574,7 @@ func (m *module) getMetricLastReceived(ctx context.Context, metricName string) (
sb := sqlbuilder.NewSelectBuilder()
sb.Select("MAX(last_reported_unix_milli) AS last_received_time")
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
sb.Where(sb.E("metric_name", metricName))
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -1599,7 +1599,7 @@ func (m *module) getTotalTimeSeriesForMetricName(ctx context.Context, metricName
sb := sqlbuilder.NewSelectBuilder()
sb.Select("uniq(fingerprint) AS time_series_count")
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV41weekTableName))
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV41weekTableName))
sb.Where(sb.E("metric_name", metricName))
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -1623,7 +1623,7 @@ func (m *module) getActiveTimeSeriesForMetricName(ctx context.Context, metricNam
sb := sqlbuilder.NewSelectBuilder()
sb.Select("uniq(fingerprint) AS active_time_series")
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4TableName))
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4TableName))
sb.Where(sb.E("metric_name", metricName))
sb.Where(sb.GTE("unix_milli", milli))
@@ -1649,7 +1649,7 @@ func (m *module) fetchMetricAttributes(ctx context.Context, metricName string, s
"groupUniqArray(1000)(attr_string_value) AS values",
"uniq(attr_string_value) AS valueCount",
)
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.AttributesMetadataTableName))
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.AttributesMetadataTableName))
sb.Where(sb.E("metric_name", metricName))
sb.Where("NOT startsWith(attr_name, '__')")

View File

@@ -9,7 +9,7 @@ import (
schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/modules/promote"
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
@@ -52,7 +52,7 @@ func (m *module) ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetype
response := []promotetypes.PromotePath{}
for _, path := range promotedPaths {
fullPath := logstelemetryschema.BodyPromotedColumnPrefix + path
fullPath := telemetrylogs.BodyPromotedColumnPrefix + path
path = telemetrytypes.BodyJSONStringSearchPrefix + path
item := promotetypes.PromotePath{
Path: path,
@@ -68,7 +68,7 @@ func (m *module) ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetype
// add the paths that are not promoted but have indexes
for path, indexes := range aggr {
path := strings.TrimPrefix(path, logstelemetryschema.BodyV2ColumnPrefix)
path := strings.TrimPrefix(path, telemetrylogs.BodyV2ColumnPrefix)
path = telemetrytypes.BodyJSONStringSearchPrefix + path
response = append(response, promotetypes.PromotePath{
Path: path,
@@ -108,8 +108,8 @@ func (m *module) createIndexes(ctx context.Context, indexes []schemamigrator.Ind
for _, index := range indexes {
alterStmt := schemamigrator.AlterTableAddIndex{
Database: logstelemetryschema.DBName,
Table: logstelemetryschema.LogsV2LocalTableName,
Database: telemetrylogs.DBName,
Table: telemetrylogs.LogsV2LocalTableName,
Index: index,
}
op := alterStmt.OnCluster(m.telemetryStore.Cluster())
@@ -153,10 +153,10 @@ func (m *module) PromoteAndIndexPaths(
}
}
if len(it.Indexes) > 0 {
parentColumn := logstelemetryschema.LogsV2BodyV2Column
parentColumn := telemetrylogs.LogsV2BodyV2Column
// if the path is already promoted or is being promoted, add it to the promoted column
if _, promoted := existingPromotedPaths[it.Path]; promoted || it.Promote {
parentColumn = logstelemetryschema.LogsV2BodyPromotedColumn
parentColumn = telemetrylogs.LogsV2BodyPromotedColumn
}
for _, index := range it.Indexes {

View File

@@ -11,8 +11,8 @@ import (
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/modules/services"
"github.com/SigNoz/signoz/pkg/querier"
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrytraces"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
@@ -39,7 +39,7 @@ func (m *module) FetchTopLevelOperations(ctx context.Context, start time.Time, s
ctx = m.withServicesContext(ctx, "FetchTopLevelOperations")
db := m.TelemetryStore.ClickhouseDB()
query := fmt.Sprintf("SELECT name, serviceName, max(time) as ts FROM %s.%s WHERE time >= @start", tracestelemetryschema.DBName, tracestelemetryschema.TopLevelOperationsTableName)
query := fmt.Sprintf("SELECT name, serviceName, max(time) as ts FROM %s.%s WHERE time >= @start", telemetrytraces.DBName, telemetrytraces.TopLevelOperationsTableName)
args := []any{clickhouse.Named("start", start)}
if len(services) > 0 {
query += " AND serviceName IN @services"

View File

@@ -0,0 +1,89 @@
package clickhouseprometheusv2
import (
"context"
"sync"
"github.com/SigNoz/signoz/pkg/prometheus"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/util/annotations"
)
// statementRecorder collects the statements a PromQL evaluation would run.
// Safe for concurrent use: the engine may Select selectors concurrently.
type statementRecorder struct {
mu sync.Mutex
statements []prometheus.CapturedStatement
}
func (r *statementRecorder) record(query string, args []any) {
r.mu.Lock()
defer r.mu.Unlock()
r.statements = append(r.statements, prometheus.CapturedStatement{Query: query, Args: args})
}
func (r *statementRecorder) Statements() []prometheus.CapturedStatement {
r.mu.Lock()
defer r.mu.Unlock()
out := make([]prometheus.CapturedStatement, len(r.statements))
copy(out, r.statements)
return out
}
type captureQueryable struct {
client *client
recorder *statementRecorder
}
func (c *captureQueryable) Querier(mint, maxt int64) (storage.Querier, error) {
return &captureQuerier{
querier: querier{mint: mint, maxt: maxt, client: c.client},
recorder: c.recorder,
}, nil
}
// captureQuerier builds the same SQL as the live querier but records it and
// returns no data. The fingerprint filter always takes the subquery form:
// without executing the series lookup, the inline literal set is unknown.
type captureQuerier struct {
querier
recorder *statementRecorder
}
func (c *captureQuerier) Select(ctx context.Context, _ bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet {
if rawQuery, ok := rawSQLQuery(matchers); ok {
c.recorder.record(rawQuery, nil)
return storage.EmptySeriesSet()
}
start, end := c.window(hints)
samplesQuery, args, err := buildSamplesQuery(start, end, metricNamesFromMatchers(matchers), nil, matchers, c.lastSamplePerStepFor(ctx, hints))
if err != nil {
return storage.ErrSeriesSet(err)
}
c.recorder.record(samplesQuery, args)
return storage.EmptySeriesSet()
}
func (c *captureQuerier) LabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
return nil, nil, nil
}
func (c *captureQuerier) LabelNames(context.Context, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
return nil, nil, nil
}
// metricNamesFromMatchers extracts the statically known metric name, if any.
// The live path derives names from the matched series; the capture path has
// no execution results, so only a __name__ equality contributes.
func metricNamesFromMatchers(matchers []*labels.Matcher) []string {
for _, m := range matchers {
if m.Name == metricNameLabel && m.Type == labels.MatchEqual && m.Value != "" {
return []string{m.Value}
}
}
return nil
}

View File

@@ -0,0 +1,282 @@
package clickhouseprometheusv2
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"log/slog"
"math"
"slices"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/prometheus"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/prometheus/prometheus/model/labels"
promValue "github.com/prometheus/prometheus/model/value"
)
// seriesLookup is a series-lookup result: matched fingerprints with their
// labels, and the distinct metric names seen on them.
type seriesLookup struct {
fingerprints map[uint64]labels.Labels
metricNames []string
}
// client executes the series, samples and raw queries against ClickHouse.
type client struct {
settings factory.ScopedProviderSettings
telemetryStore telemetrystore.TelemetryStore
cfg prometheus.ClickhouseV2Config
lookbackMs int64
}
func newClient(settings factory.ScopedProviderSettings, telemetryStore telemetrystore.TelemetryStore, cfg prometheus.Config) *client {
lookback := cfg.LookbackDelta
if lookback <= 0 {
// Mirror the engine: promql defaults an unset lookback to 5m.
lookback = defaultLookbackDelta
}
return &client{
settings: settings,
telemetryStore: telemetryStore,
cfg: cfg.ClickhouseV2,
lookbackMs: lookback.Milliseconds(),
}
}
func (c *client) withContext(ctx context.Context, functionName string) context.Context {
return ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalMetrics.StringValue(),
instrumentationtypes.CodeNamespace: "clickhouse-prometheus-v2",
instrumentationtypes.CodeFunctionName: functionName,
})
}
// selectSeries runs the series lookup for the given matchers and window.
func (c *client) selectSeries(ctx context.Context, query string, args []any) (*seriesLookup, error) {
ctx = c.withContext(ctx, "selectSeries")
rows, err := c.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
lookup := &seriesLookup{fingerprints: make(map[uint64]labels.Labels)}
names := make(map[string]struct{})
var fingerprint uint64
var labelsJSON string
for rows.Next() {
if err := rows.Scan(&fingerprint, &labelsJSON); err != nil {
return nil, err
}
lset, err := unmarshalLabels(labelsJSON)
if err != nil {
return nil, err
}
lookup.fingerprints[fingerprint] = lset
if name := lset.Get(metricNameLabel); name != "" {
names[name] = struct{}{}
}
if c.cfg.MaxFetchedSeries > 0 && len(lookup.fingerprints) > c.cfg.MaxFetchedSeries {
return nil, errors.NewInvalidInputf(
errors.CodeInvalidInput,
"promql selector matched more than %d series; narrow the label matchers or raise prometheus::clickhousev2::max_fetched_series",
c.cfg.MaxFetchedSeries,
)
}
}
if err := rows.Err(); err != nil {
return nil, err
}
for name := range names {
lookup.metricNames = append(lookup.metricNames, name)
}
slices.Sort(lookup.metricNames)
return lookup, nil
}
// unmarshalLabels parses the labels JSON column. Unlike v1, the fingerprint
// is not injected as a synthetic label (it would take part in `without (...)`
// grouping and vector matching) and empty-valued labels are dropped: an empty
// label value means "label absent" in Prometheus, and upstream never produces
// such labels, but stored attribute JSON can carry them.
func unmarshalLabels(s string) (labels.Labels, error) {
m := make(map[string]string)
if err := json.Unmarshal([]byte(s), &m); err != nil {
return labels.EmptyLabels(), err
}
builder := labels.NewScratchBuilder(len(m))
for k, v := range m {
if v == "" {
continue
}
builder.Add(k, v)
}
builder.Sort()
return builder.Labels(), nil
}
// selectSamples executes a samples query (raw or last-sample-per-step; both
// produce the same column shape) and assembles the per-series sample slices.
// Rows arrive ordered by (fingerprint, unix_milli). Rows whose fingerprint
// is missing from the lookup are skipped (possible in the subquery filter
// mode, where the fingerprint filter re-runs after the lookup and can see
// series born in between). Stale flags map to the engine's StaleNaN.
// Duplicate timestamps pass through as stored: upstream Prometheus cannot
// produce them (its TSDB rejects them at ingest), our ingest can under
// at-least-once retries, and v1 feeds them to the engine as-is —
// deduplicating here would make this provider silently disagree with both
// v1 and the transpiled statements over the same dirty data. Uniqueness
// belongs to the ingest layer.
func (c *client) selectSamples(ctx context.Context, query string, args []any, lookup *seriesLookup) ([]*series, error) {
ctx = c.withContext(ctx, "selectSamples")
rows, err := c.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var (
result []*series
current *series
fingerprint uint64
prevFp uint64
timestampMs int64
val float64
flags uint32
first = true
haveCurrent bool
staleMarker = math.Float64frombits(promValue.StaleNaN)
maxSamples = c.cfg.MaxFetchedSamples
fetched int64
unknownCount int
)
for rows.Next() {
if err := rows.Scan(&fingerprint, &timestampMs, &val, &flags); err != nil {
return nil, err
}
fetched++
if maxSamples > 0 && fetched > maxSamples {
return nil, errors.NewInvalidInputf(
errors.CodeInvalidInput,
"promql query would fetch more than %d samples; narrow the selector or time range, or raise prometheus::clickhousev2::max_fetched_samples",
maxSamples,
)
}
if first || fingerprint != prevFp {
first = false
prevFp = fingerprint
lset, ok := lookup.fingerprints[fingerprint]
if !ok {
unknownCount++
haveCurrent = false
continue
}
current = &series{lset: lset}
result = append(result, current)
haveCurrent = true
}
if !haveCurrent {
// Remaining rows of a fingerprint missing from the lookup.
continue
}
if flags&1 == 1 {
val = staleMarker
}
current.ts = append(current.ts, timestampMs)
current.vs = append(current.vs, val)
}
if err := rows.Err(); err != nil {
return nil, err
}
if unknownCount > 0 {
c.settings.Logger().DebugContext(ctx, "skipped samples of fingerprints missing from series lookup",
slog.Int("unknown_fingerprints", unknownCount))
}
return result, nil
}
// queryRaw supports the {job="rawsql", query="..."} escape hatch: the value of
// the query matcher runs as-is, each row becoming a single-sample series
// stamped at the query end. Column "value" is the sample value; every other
// column is a label.
func (c *client) queryRaw(ctx context.Context, query string, ts int64) ([]*series, error) {
ctx = c.withContext(ctx, "queryRaw")
rows, err := c.telemetryStore.ClickhouseDB().Query(ctx, query)
if err != nil {
return nil, err
}
defer rows.Close()
columns := rows.Columns()
targets := make([]any, len(columns))
for i := range targets {
targets[i] = new(scanner)
}
var result []*series
for rows.Next() {
if err := rows.Scan(targets...); err != nil {
return nil, err
}
builder := labels.NewScratchBuilder(len(columns))
var val float64
for i, col := range columns {
v := targets[i].(*scanner)
if col == "value" {
val = v.f
continue
}
builder.Add(col, v.s)
}
builder.Sort()
result = append(result, &series{
lset: builder.Labels(),
ts: []int64{ts},
vs: []float64{val},
})
}
if err := rows.Err(); err != nil {
return nil, err
}
return result, nil
}
var _ sql.Scanner = (*scanner)(nil)
type scanner struct {
f float64
s string
}
func (s *scanner) Scan(val any) error {
s.f = 0
s.s = ""
s.s = fmt.Sprintf("%v", val)
switch val := val.(type) {
case int64:
s.f = float64(val)
case uint64:
s.f = float64(val)
case float64:
s.f = val
case []byte:
s.s = string(val)
}
return nil
}

View File

@@ -0,0 +1,86 @@
// Package clickhouseprometheusv2 is the second-generation ClickHouse-backed
// Prometheus provider. It exists because the v1 provider fetches every raw
// sample of a query's union window through the remote-read protobuf layer
// and hands it to the engine — the cost is a function of ingested data, not
// of the question asked, which is how a dashboard of PromQL panels takes an
// instance down.
//
// Here the stock promql engine evaluates over this package's native
// storage.Querier: no remote-read translation layer, per-selector fetch
// windows instead of the query-wide union, and fetch reductions that are
// provably invisible to the engine. Correctness is the constraint that
// shaped everything: a PromQL result that differs from upstream Prometheus
// is a lost user, so every reduction below either preserves engine semantics
// exactly or is not performed.
//
// # Series lookup
//
// Matchers resolve to series once per selector (selectSeries): __name__
// matchers translate to the metric_name column — all four matcher types; the
// v1 client silently returned nothing for regex metric names — and every
// other matcher to a JSONExtractString condition on the labels column
// (applySeriesConditions). Regexes are anchored before they reach match():
// PromQL matchers match the whole value, ClickHouse match() searches for a
// substring, and without anchoring =~"api" would also select "x-api-y". An
// equality matcher against "" matches series without the label, mirroring
// PromQL, because JSONExtractString returns "" for missing keys. The series
// tables hold one row per (fingerprint, bucket) at 1h/6h/1d/1w
// granularities; timeSeriesTableFor picks the table whose bucket fits the
// window and rounds the window start down to the bucket boundary. The
// resulting label sets drop what v1 leaked into results: the synthetic
// fingerprint label (it would take part in without() grouping and vector
// matching) and empty-valued labels. MaxFetchedSeries fails the lookup with
// a typed invalid-input error past the ceiling — v1's behavior for an
// oversized selector was to buffer everything and OOM, and a 4xx the user
// can narrow beats a dead process serving nobody.
//
// # Sample fetch
//
// Samples are fetched per selector using the engine's per-selector hints,
// not the query-wide union window, so foo / foo offset 1d reads two narrow
// windows instead of the widest one twice. Instant selectors of
// subquery-free queries fetch only the last sample per step bucket
// (lastSamplePerStep): the engine resolves an instant selector at each grid
// timestamp t to the latest sample in the left-open lookback window
// (t - lookback, t], so buckets anchor at the selector's first evaluation
// timestamp — recovered from the hints as hints.Start + lookback - 1ms, the
// inverse of how the engine derives hints.Start — and bucket boundaries
// coincide with evaluation timestamps: a non-final sample of a bucket can
// never be the latest sample in (t - lookback, t] for any grid t. Real
// timestamps are preserved, so the engine's own lookback and staleness
// handling stay exact. Range selectors always fetch raw — every sample feeds
// the range function — and the subquery-free proof travels in the context as
// prometheus.QueryTraits, because subquery selectors evaluate at the
// subquery's step while the hints carry the top-level step; call sites that
// do not attach traits get the conservative raw fetch. Row assembly counts
// rows against MaxFetchedSamples while scanning, keeps the first of
// consecutive equal timestamps, maps stale flags to the engine's StaleNaN,
// and merges series with identical label sets (sortAndMerge) — the engine
// assumes storages never emit duplicates. A {job="rawsql", query="..."}
// selector bypasses all of this and runs the query matcher's value verbatim.
//
// # Sharding
//
// samples_v4 and time_series_v4 (and all their rollups) shard on the same
// key — cityHash64(env, temporality, metric_name, fingerprint) — so a
// series' samples and catalog rows live on the same shard. Fingerprint
// filters exploit that: matched sets inline as sorted literals up to
// inlineFingerprintsLimit (literals engage the samples primary key; sorting
// keeps statements deterministic), beyond it the filter is a shard-local
// series subquery on the same predicates, not a GLOBAL broadcast of the
// matched set. The temporality filter on every samples statement is a
// semantic no-op — the matched fingerprints already come from those
// temporalities — that engages the leading samples primary-key column.
// Delta-temporality series stay invisible to PromQL here exactly as they are
// in v1: the rollout gate is parity with v1, and making Delta visible is its
// own change with its own semantics to design — a Delta stream fed to rate()
// as-if-cumulative would be wrong, not just new.
//
// # Observability
//
// Every statement carries a log_comment with
// code.namespace=clickhouse-prometheus-v2 and code.function.name naming the
// call site (selectSeries, selectSamples, LabelValues, LabelNames), so this
// provider's work is attributable in system.query_log without guessing from
// query text.
package clickhouseprometheusv2

View File

@@ -0,0 +1,191 @@
package clickhouseprometheusv2
import (
"fmt"
"math/rand"
"sort"
"testing"
"github.com/stretchr/testify/require"
)
// The lastSamplePerStep correctness argument, executed: for instant selectors, keeping
// only the last sample of every step bucket (bucket 0 = (start, firstEval],
// bucket i = (firstEval+(i-1)·step, firstEval+i·step]) yields exactly the
// same instant-vector selections as the raw samples, for every evaluation
// timestamp on the grid. The engine picks the latest sample in
// (t-lookback, t] per evaluation timestamp t and treats a stale marker as
// absent; both behaviors are emulated here directly.
type tsample struct {
ts int64
value float64
stale bool
}
// engineSelect emulates the engine's instant-selector resolution at
// evaluation timestamp t over samples ordered by timestamp: the latest sample
// in (t-lookback, t], absent when none or when it is a stale marker.
func engineSelect(samples []tsample, t, lookbackMs int64) (tsample, bool) {
var picked tsample
found := false
for _, s := range samples {
if s.ts > t-lookbackMs && s.ts <= t {
picked = s
found = true
}
}
if !found || picked.stale {
return tsample{}, false
}
return picked, true
}
// lastPerStep emulates the last-sample-per-step samples query: group samples into buckets and
// keep only the last sample of each (ties keep either; ClickHouse argMax over
// equal keys is unspecified, so generated timestamps are unique).
func lastPerStep(samples []tsample, firstEvalMs, stepMs int64) []tsample {
last := make(map[int64]tsample)
for _, s := range samples {
var bucket int64
if stepMs > 0 && s.ts > firstEvalMs {
bucket = (s.ts-firstEvalMs-1)/stepMs + 1
}
if cur, ok := last[bucket]; !ok || s.ts > cur.ts {
last[bucket] = s
}
}
out := make([]tsample, 0, len(last))
for _, s := range last {
out = append(out, s)
}
sort.Slice(out, func(i, j int) bool { return out[i].ts < out[j].ts })
return out
}
func TestLastSamplePerStepEquivalence(t *testing.T) {
rng := rand.New(rand.NewSource(42))
for caseIdx := 0; caseIdx < 2000; caseIdx++ {
// Random query shape. Units are milliseconds but kept small so bucket
// boundaries are hit often.
stepMs := []int64{1, 2, 5, 7, 30, 60}[rng.Intn(6)]
lookbackMs := []int64{1, 3, 5, 10, 45}[rng.Intn(5)]
queryStart := int64(1000)
numSteps := rng.Int63n(20)
queryEnd := queryStart + numSteps*stepMs + rng.Int63n(stepMs) // grid may not divide the range
// Engine-derived selector window for instant selectors:
// hints.Start = firstEval - (lookback - 1), hints.End = queryEnd.
hintsStart := queryStart - (lookbackMs - 1)
hintsEnd := queryEnd
firstEval := hintsStart + lookbackMs - 1
require.Equal(t, queryStart, firstEval)
// Random samples inside the fetch window [hints.Start, hints.End],
// with unique timestamps and occasional stale markers. The sample
// count is capped by the window size: timestamps are unique.
windowSize := hintsEnd - hintsStart + 1
numSamples := rng.Int63n(40)
if numSamples > windowSize {
numSamples = windowSize
}
seen := make(map[int64]bool)
var samples []tsample
for int64(len(samples)) < numSamples {
ts := hintsStart + rng.Int63n(windowSize)
if seen[ts] {
continue
}
seen[ts] = true
samples = append(samples, tsample{ts: ts, value: rng.Float64(), stale: rng.Intn(8) == 0})
}
sort.Slice(samples, func(i, j int) bool { return samples[i].ts < samples[j].ts })
reduced := lastPerStep(samples, firstEval, stepMs)
desc := fmt.Sprintf("case=%d step=%d lookback=%d start=%d end=%d samples=%d",
caseIdx, stepMs, lookbackMs, queryStart, queryEnd, len(samples))
for evalTs := queryStart; evalTs <= queryEnd; evalTs += stepMs {
rawPick, rawOK := engineSelect(samples, evalTs, lookbackMs)
reducedPick, reducedOK := engineSelect(reduced, evalTs, lookbackMs)
require.Equal(t, rawOK, reducedOK, "%s eval=%d presence mismatch", desc, evalTs)
if rawOK {
require.Equal(t, rawPick, reducedPick, "%s eval=%d sample mismatch", desc, evalTs)
}
}
}
}
// Instant queries (step 0) evaluate once at firstEval == hints.End; lastSamplePerStep
// collapses to a single bucket over the whole window.
func TestLastSamplePerStepEquivalenceInstantQuery(t *testing.T) {
rng := rand.New(rand.NewSource(7))
for caseIdx := 0; caseIdx < 500; caseIdx++ {
lookbackMs := []int64{1, 3, 5, 10, 45}[rng.Intn(5)]
evalTs := int64(1000)
hintsStart := evalTs - (lookbackMs - 1)
hintsEnd := evalTs
firstEval := hintsStart + lookbackMs - 1
require.Equal(t, evalTs, firstEval)
windowSize := hintsEnd - hintsStart + 1
numSamples := rng.Int63n(10)
if numSamples > windowSize {
numSamples = windowSize
}
seen := make(map[int64]bool)
var samples []tsample
for int64(len(samples)) < numSamples {
ts := hintsStart + rng.Int63n(windowSize)
if seen[ts] {
continue
}
seen[ts] = true
samples = append(samples, tsample{ts: ts, value: rng.Float64(), stale: rng.Intn(4) == 0})
}
sort.Slice(samples, func(i, j int) bool { return samples[i].ts < samples[j].ts })
reduced := lastPerStep(samples, firstEval, 0)
require.LessOrEqual(t, len(reduced), 1, "instant reduction must keep at most one sample")
rawPick, rawOK := engineSelect(samples, evalTs, lookbackMs)
reducedPick, reducedOK := engineSelect(reduced, evalTs, lookbackMs)
require.Equal(t, rawOK, reducedOK, "case=%d presence mismatch", caseIdx)
if rawOK {
require.Equal(t, rawPick, reducedPick, "case=%d sample mismatch", caseIdx)
}
}
}
// A stale marker that is the latest sample of its bucket must shadow older
// samples: the engine sees the marker and reports the series absent, exactly
// as with raw samples. Pre-filtering stale rows would instead resurrect the
// older sample.
func TestLastSamplePerStepKeepsStaleShadowing(t *testing.T) {
lookbackMs := int64(10)
stepMs := int64(5)
queryStart := int64(1000)
samples := []tsample{
{ts: 998, value: 1.0}, // bucket 0
{ts: 999, stale: true}, // bucket 0: marker shadows 998
{ts: 1003, value: 2.0}, // bucket 1
{ts: 1004, stale: true}, // bucket 1: marker shadows 1003
{ts: 1008, value: 3.0, stale: false}, // bucket 2
}
firstEval := queryStart
reduced := lastPerStep(samples, firstEval, stepMs)
for evalTs := queryStart; evalTs <= queryStart+2*stepMs; evalTs += stepMs {
rawPick, rawOK := engineSelect(samples, evalTs, lookbackMs)
reducedPick, reducedOK := engineSelect(reduced, evalTs, lookbackMs)
require.Equal(t, rawOK, reducedOK, "eval=%d", evalTs)
if rawOK {
require.Equal(t, rawPick, reducedPick, "eval=%d", evalTs)
}
}
}

View File

@@ -0,0 +1,72 @@
package clickhouseprometheusv2
import (
"context"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/prometheus"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/prometheus/prometheus/storage"
)
// Provider ties the package together: its own engine and parser, and the
// ClickHouse client behind the native storage.Querier. See the package
// documentation for how the read path differs from v1. It is exported as a
// concrete type — callers hold it directly, and an interface with a single
// implementation would only hide that dependency.
type Provider struct {
settings factory.ScopedProviderSettings
engine *prometheus.Engine
parser prometheus.Parser
client *client
}
var (
_ prometheus.Prometheus = (*Provider)(nil)
_ prometheus.StatementCapturer = (*Provider)(nil)
)
func NewFactory(telemetryStore telemetrystore.TelemetryStore) factory.ProviderFactory[prometheus.Prometheus, prometheus.Config] {
return factory.NewProviderFactory(factory.MustNewName("clickhousev2"), func(ctx context.Context, providerSettings factory.ProviderSettings, config prometheus.Config) (prometheus.Prometheus, error) {
return New(ctx, providerSettings, config, telemetryStore)
})
}
func New(_ context.Context, providerSettings factory.ProviderSettings, config prometheus.Config, telemetryStore telemetrystore.TelemetryStore) (*Provider, error) {
settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2")
engine := prometheus.NewEngine(settings.Logger(), config)
parser := prometheus.NewParser()
client := newClient(settings, telemetryStore, config)
return &Provider{
settings: settings,
engine: engine,
parser: parser,
client: client,
}, nil
}
func (p *Provider) Engine() *prometheus.Engine {
return p.engine
}
func (p *Provider) Parser() prometheus.Parser {
return p.parser
}
func (p *Provider) Storage() storage.Queryable {
return p
}
func (p *Provider) Querier(mint, maxt int64) (storage.Querier, error) {
return &querier{mint: mint, maxt: maxt, client: p.client}, nil
}
// CapturingStorage implements prometheus.StatementCapturer: a storage that
// records each selector's SQL without executing it, for the preview path.
// A fresh recorder per call keeps concurrent dry-runs isolated.
func (p *Provider) CapturingStorage() (storage.Queryable, prometheus.StatementRecorder) {
recorder := &statementRecorder{}
return &captureQueryable{client: p.client, recorder: recorder}, recorder
}

View File

@@ -0,0 +1,233 @@
package clickhouseprometheusv2
import (
"context"
"fmt"
"slices"
"sort"
"time"
"github.com/SigNoz/signoz/pkg/prometheus"
"github.com/huandu/go-sqlbuilder"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/util/annotations"
)
// defaultLookbackDelta mirrors promql's default when the config leaves the
// lookback unset; the engine and the storage must agree on it for
// last-sample-per-step bucket anchoring.
const defaultLookbackDelta = 5 * time.Minute
// querier is a native storage.Querier over ClickHouse. Unlike v1 it does not
// round-trip through the remote-read protobuf machinery: Select builds SQL
// directly from the matchers and hints, and the result set is assembled once
// into compact series.
type querier struct {
mint, maxt int64
client *client
}
var _ storage.Querier = (*querier)(nil)
func (q *querier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet {
if rawQuery, ok := rawSQLQuery(matchers); ok {
_, end := q.window(hints)
list, err := q.client.queryRaw(ctx, rawQuery, end)
if err != nil {
return storage.ErrSeriesSet(err)
}
if sortSeries {
sort.Slice(list, func(i, j int) bool { return labels.Compare(list[i].lset, list[j].lset) < 0 })
}
return newSeriesSet(list)
}
start, end := q.window(hints)
seriesQuery, seriesArgs, err := buildSeriesQuery(start, end, matchers)
if err != nil {
return storage.ErrSeriesSet(err)
}
lookup, err := q.client.selectSeries(ctx, seriesQuery, seriesArgs)
if err != nil {
return storage.ErrSeriesSet(err)
}
if len(lookup.fingerprints) == 0 {
return storage.EmptySeriesSet()
}
list, err := q.fetchSamples(ctx, start, end, matchers, lookup, q.lastSamplePerStepFor(ctx, hints))
if err != nil {
return storage.ErrSeriesSet(err)
}
// Sorting doubles as duplicate-label-set detection, which the engine
// depends on storages never emitting; the cost is on series count, not
// samples.
list = sortAndMerge(list)
return newSeriesSet(list)
}
// LabelValues returns the values of a label across series matching the
// matchers within the querier window.
func (q *querier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
sb := sqlbuilder.NewSelectBuilder()
if name == metricNameLabel {
sb.Select("DISTINCT metric_name AS value")
} else {
sb.Select(fmt.Sprintf("DISTINCT JSONExtractString(labels, %s) AS value", sb.Var(name)))
}
adjustedStart, table := timeSeriesTableFor(q.mint, q.maxt)
sb.From(fmt.Sprintf("%s.%s", databaseName, table))
if err := applySeriesConditions(sb, adjustedStart, q.maxt, matchers); err != nil {
return nil, nil, err
}
sb.Where("value != ''")
if hints != nil && hints.Limit > 0 {
sb.Limit(hints.Limit)
}
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
values, err := q.selectStrings(ctx, "LabelValues", query, args)
if err != nil {
return nil, nil, err
}
slices.Sort(values)
return values, nil, nil
}
// LabelNames returns the label names present on series matching the matchers
// within the querier window.
func (q *querier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
sb := sqlbuilder.NewSelectBuilder()
sb.Select("DISTINCT arrayJoin(JSONExtractKeys(labels)) AS name")
adjustedStart, table := timeSeriesTableFor(q.mint, q.maxt)
sb.From(fmt.Sprintf("%s.%s", databaseName, table))
if err := applySeriesConditions(sb, adjustedStart, q.maxt, matchers); err != nil {
return nil, nil, err
}
if hints != nil && hints.Limit > 0 {
sb.Limit(hints.Limit)
}
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
names, err := q.selectStrings(ctx, "LabelNames", query, args)
if err != nil {
return nil, nil, err
}
slices.Sort(names)
return names, nil, nil
}
func (q *querier) Close() error {
return nil
}
// window returns the per-selector fetch window. The engine sends per-selector
// bounds in the hints (already adjusted for offset, @, range and lookback);
// they are always at least as tight as the querier-level mint/maxt, which
// span the union of all selectors in the query.
func (q *querier) window(hints *storage.SelectHints) (int64, int64) {
if hints != nil && hints.Start != 0 && hints.End != 0 && hints.Start <= hints.End {
return hints.Start, hints.End
}
return q.mint, q.maxt
}
// lastSamplePerStepFor decides whether the fetch can keep only the last
// sample per step bucket, and computes the bucket parameters. Requirements:
// - the call site attached QueryTraits proving the query has no subquery
// (subquery selectors evaluate at the subquery's own step, but hints
// carry the top-level step);
// - the selector is an instant selector (hints.Range == 0); range selectors
// need every raw sample in the window;
// - per-selector hints are present.
//
// The engine derives hints.Start for instant selectors as
// firstEval - (lookback - 1ms), so the first evaluation timestamp is
// recovered as hints.Start + lookback - 1ms. Bucket boundaries then coincide
// with evaluation timestamps, which is what makes keeping only the last
// sample per bucket lossless.
func (q *querier) lastSamplePerStepFor(ctx context.Context, hints *storage.SelectHints) *lastSamplePerStep {
if hints == nil || hints.Range != 0 || hints.Start <= 0 {
return nil
}
traits, ok := prometheus.QueryTraitsFromContext(ctx)
if !ok || !traits.SubqueryFree {
return nil
}
firstEval := hints.Start + q.client.lookbackMs - 1
if firstEval > hints.End {
// Defensive: never anchor a bucket past the window.
firstEval = hints.End
}
return &lastSamplePerStep{firstEvalMs: firstEval, stepMs: hints.Step}
}
// fetchSamples runs the samples query for the matched series. Small sets
// inline the fingerprints as sorted uint64 literals — literals engage the
// samples primary key, and sorting keeps the statement deterministic for
// logging and tests. Larger sets re-run the series predicates as a
// shard-local IN subquery instead: inlining hundreds of thousands of
// literals makes the statement itself the bottleneck, while the subquery is
// a cheap primary-key scan on each shard's own series table (see
// localTimeSeriesTable for why that is complete).
func (q *querier) fetchSamples(ctx context.Context, start, end int64, matchers []*labels.Matcher, lookup *seriesLookup, lastPerStep *lastSamplePerStep) ([]*series, error) {
var fingerprints []uint64
if len(lookup.fingerprints) <= inlineFingerprintsLimit {
fingerprints = make([]uint64, 0, len(lookup.fingerprints))
for fp := range lookup.fingerprints {
fingerprints = append(fingerprints, fp)
}
slices.Sort(fingerprints)
}
query, args, err := buildSamplesQuery(start, end, lookup.metricNames, fingerprints, matchers, lastPerStep)
if err != nil {
return nil, err
}
return q.client.selectSamples(ctx, query, args, lookup)
}
func (q *querier) selectStrings(ctx context.Context, fn, query string, args []any) ([]string, error) {
ctx = q.client.withContext(ctx, fn)
rows, err := q.client.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []string
var v string
for rows.Next() {
if err := rows.Scan(&v); err != nil {
return nil, err
}
out = append(out, v)
}
if err := rows.Err(); err != nil {
return nil, err
}
return out, nil
}
// rawSQLQuery detects the {job="rawsql", query="..."} escape hatch.
func rawSQLQuery(matchers []*labels.Matcher) (string, bool) {
if len(matchers) != 2 {
return "", false
}
var hasJob bool
var query string
for _, m := range matchers {
if m.Type == labels.MatchEqual && m.Name == "job" && m.Value == "rawsql" {
hasJob = true
}
if m.Type == labels.MatchEqual && m.Name == "query" {
query = m.Value
}
}
if hasJob && query != "" {
return query, true
}
return "", false
}

View File

@@ -0,0 +1,221 @@
package clickhouseprometheusv2
import (
"context"
"fmt"
"testing"
"github.com/DATA-DOG/go-sqlmock"
cmock "github.com/SigNoz/clickhouse-go-mock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/prometheus"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/storage"
)
var (
seriesCols = []cmock.ColumnType{
{Name: "fingerprint", Type: "UInt64"},
{Name: "labels", Type: "String"},
}
samplesCols = []cmock.ColumnType{
{Name: "fingerprint", Type: "UInt64"},
{Name: "unix_milli", Type: "Int64"},
{Name: "value", Type: "Float64"},
{Name: "flags", Type: "UInt32"},
}
)
func newTestClient(t *testing.T, cfg prometheus.ClickhouseV2Config) (*client, *telemetrystoretest.Provider) {
t.Helper()
store := telemetrystoretest.New(telemetrystore.Config{Provider: "clickhouse"}, sqlmock.QueryMatcherRegexp)
settings := factory.NewScopedProviderSettings(instrumentationtest.New().ToProviderSettings(), "clickhouseprometheusv2_test")
promCfg := prometheus.Config{ClickhouseV2: cfg}
return newClient(settings, store, promCfg), store
}
func testMatchers(t *testing.T) []*labels.Matcher {
t.Helper()
return []*labels.Matcher{
mustMatcher(t, labels.MatchEqual, "__name__", "cpu_usage"),
mustMatcher(t, labels.MatchEqual, "job", "api"),
}
}
func TestQuerierSelectRawPath(t *testing.T) {
c, store := newTestClient(t, prometheus.ClickhouseV2Config{})
q := &querier{mint: 1000, maxt: 2000, client: c}
store.Mock().ExpectQuery("SELECT fingerprint, any\\(labels\\)").WithArgs("cpu_usage", int64(0), int64(2000), "job", "api").WillReturnRows(cmock.NewRows(seriesCols, [][]any{
{uint64(42), `{"__name__":"cpu_usage","job":"api","instance":"a"}`},
{uint64(7), `{"__name__":"cpu_usage","job":"api","instance":"b"}`},
}))
// Inline fingerprints (sorted), raw samples: no traits in ctx -> no
// last-sample-per-step reduction.
store.Mock().ExpectQuery("SELECT fingerprint, unix_milli, value, flags FROM signoz_metrics.distributed_samples_v4 WHERE metric_name = \\? AND temporality IN \\['Cumulative', 'Unspecified'\\] AND fingerprint IN \\(7, 42\\)").
WithArgs("cpu_usage", int64(1000), int64(2000)).
WillReturnRows(cmock.NewRows(samplesCols, [][]any{
{uint64(7), int64(1100), 1.5, uint32(0)},
{uint64(7), int64(1200), 2.5, uint32(0)},
{uint64(42), int64(1100), 3.5, uint32(1)}, // stale marker
}))
hints := &storage.SelectHints{Start: 1000, End: 2000, Step: 60_000}
set := q.Select(context.Background(), false, hints, testMatchers(t)...)
var got []*series
for set.Next() {
got = append(got, set.At().(*series))
}
require.NoError(t, set.Err())
require.Len(t, got, 2)
// Sorted by labels: instance=a (fp 42) before instance=b (fp 7).
assert.Equal(t, "a", got[0].lset.Get("instance"))
require.Len(t, got[0].ts, 1)
assert.True(t, got[0].vs[0] != got[0].vs[0], "stale marker must be NaN") //nolint:testifylint
assert.Equal(t, "b", got[1].lset.Get("instance"))
assert.Equal(t, []int64{1100, 1200}, got[1].ts)
assert.Equal(t, []float64{1.5, 2.5}, got[1].vs)
// No fingerprint label injected.
assert.Empty(t, got[0].lset.Get("fingerprint"))
}
// Wrong gating silently corrupts range functions (a rate over reduced
// samples loses points), so the decision logic is pinned here even though
// the helper is unexported: the integration suite would catch it too, but
// with far worse failure locality.
func TestLastSamplePerStepFor(t *testing.T) {
c, _ := newTestClient(t, prometheus.ClickhouseV2Config{})
q := &querier{mint: 0, maxt: 2000, client: c}
traitsCtx := prometheus.NewContextWithQueryTraits(context.Background(), prometheus.QueryTraits{SubqueryFree: true})
tests := []struct {
name string
ctx context.Context
hints *storage.SelectHints
want *lastSamplePerStep
}{
{"no traits in context stays raw", context.Background(), &storage.SelectHints{Start: 1000, End: 2000, Step: 60_000}, nil},
{"subquery in the query stays raw", prometheus.NewContextWithQueryTraits(context.Background(), prometheus.QueryTraits{SubqueryFree: false}), &storage.SelectHints{Start: 1000, End: 2000, Step: 60_000}, nil},
{"range selector stays raw", traitsCtx, &storage.SelectHints{Start: 1000, End: 2000, Step: 60_000, Range: 300_000}, nil},
{"instant selector reduces, anchored at first eval", traitsCtx, &storage.SelectHints{Start: 1000, End: 2_000_000, Step: 60_000}, &lastSamplePerStep{firstEvalMs: 1000 + c.lookbackMs - 1, stepMs: 60_000}},
{"anchor never passes the window end", traitsCtx, &storage.SelectHints{Start: 1000, End: 2000, Step: 60_000}, &lastSamplePerStep{firstEvalMs: 2000, stepMs: 60_000}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, q.lastSamplePerStepFor(tt.ctx, tt.hints))
})
}
}
func TestQuerierSelectSeriesBudget(t *testing.T) {
c, store := newTestClient(t, prometheus.ClickhouseV2Config{MaxFetchedSeries: 1})
q := &querier{mint: 1000, maxt: 2000, client: c}
store.Mock().ExpectQuery("SELECT fingerprint, any\\(labels\\)").WithArgs("cpu_usage", int64(0), int64(2000), "job", "api").WillReturnRows(cmock.NewRows(seriesCols, [][]any{
{uint64(1), `{"__name__":"cpu_usage","instance":"a"}`},
{uint64(2), `{"__name__":"cpu_usage","instance":"b"}`},
}))
set := q.Select(context.Background(), false, &storage.SelectHints{Start: 1000, End: 2000}, testMatchers(t)...)
assert.False(t, set.Next())
require.Error(t, set.Err())
assert.True(t, errors.Ast(set.Err(), errors.TypeInvalidInput), "budget error must be typed invalid input, got %v", set.Err())
}
func TestQuerierSelectSamplesBudget(t *testing.T) {
c, store := newTestClient(t, prometheus.ClickhouseV2Config{MaxFetchedSamples: 2})
q := &querier{mint: 1000, maxt: 2000, client: c}
store.Mock().ExpectQuery("SELECT fingerprint, any\\(labels\\)").WithArgs("cpu_usage", int64(0), int64(2000)).WillReturnRows(cmock.NewRows(seriesCols, [][]any{
{uint64(7), `{"__name__":"cpu_usage"}`},
}))
store.Mock().ExpectQuery("SELECT fingerprint, unix_milli, value, flags").
WithArgs("cpu_usage", int64(1000), int64(2000)).
WillReturnRows(cmock.NewRows(samplesCols, [][]any{
{uint64(7), int64(1100), 1.0, uint32(0)},
{uint64(7), int64(1200), 2.0, uint32(0)},
{uint64(7), int64(1300), 3.0, uint32(0)},
}))
set := q.Select(context.Background(), false, &storage.SelectHints{Start: 1000, End: 2000},
mustMatcher(t, labels.MatchEqual, "__name__", "cpu_usage"))
assert.False(t, set.Next())
require.Error(t, set.Err())
assert.True(t, errors.Ast(set.Err(), errors.TypeInvalidInput))
}
func TestQuerierSelectSubqueryFilterOverInlineLimit(t *testing.T) {
c, store := newTestClient(t, prometheus.ClickhouseV2Config{})
q := &querier{mint: 1000, maxt: 2000, client: c}
seriesRows := make([][]any, inlineFingerprintsLimit+1)
for i := range seriesRows {
seriesRows[i] = []any{uint64(i + 1), fmt.Sprintf(`{"__name__":"cpu_usage","instance":"i%d"}`, i)}
}
store.Mock().ExpectQuery("SELECT fingerprint, any\\(labels\\)").WithArgs("cpu_usage", int64(0), int64(2000), "job", "api").WillReturnRows(cmock.NewRows(seriesCols, seriesRows))
// The over-limit samples query embeds the semi-join against the
// shard-local series table (fingerprint co-locality), not a GLOBAL
// broadcast; args follow placeholder order — samples metric name, then
// the semi-join's series predicates, then the samples window bounds.
store.Mock().ExpectQuery("fingerprint IN \\(SELECT fingerprint FROM signoz_metrics\\.time_series_v4").
WithArgs("cpu_usage", "cpu_usage", int64(0), int64(2000), "job", "api", int64(1000), int64(2000)).
WillReturnRows(cmock.NewRows(samplesCols, [][]any{}))
set := q.Select(context.Background(), false, &storage.SelectHints{Start: 1000, End: 2000}, testMatchers(t)...)
assert.False(t, set.Next())
require.NoError(t, set.Err())
}
func TestQuerierSelectRawSQLPassthrough(t *testing.T) {
c, store := newTestClient(t, prometheus.ClickhouseV2Config{})
q := &querier{mint: 1000, maxt: 2000, client: c}
rawCols := []cmock.ColumnType{
{Name: "le", Type: "String"},
{Name: "value", Type: "Float64"},
}
store.Mock().ExpectQuery("SELECT le, avg\\(v\\) AS value FROM t").WillReturnRows(cmock.NewRows(rawCols, [][]any{
{"0.5", 12.5},
}))
set := q.Select(context.Background(), false, &storage.SelectHints{Start: 1000, End: 2000},
mustMatcher(t, labels.MatchEqual, "job", "rawsql"),
mustMatcher(t, labels.MatchEqual, "query", "SELECT le, avg(v) AS value FROM t"),
)
require.True(t, set.Next())
s := set.At()
assert.Equal(t, "0.5", s.Labels().Get("le"))
it := s.Iterator(nil)
require.NotNil(t, it)
_, v := func() (int64, float64) { it.Next(); return it.At() }()
assert.Equal(t, 12.5, v)
assert.False(t, set.Next())
}
func TestCaptureQuerierRecordsWithoutExecuting(t *testing.T) {
c, _ := newTestClient(t, prometheus.ClickhouseV2Config{})
recorder := &statementRecorder{}
cq := &captureQuerier{querier: querier{mint: 1000, maxt: 2000, client: c}, recorder: recorder}
ctx := prometheus.NewContextWithQueryTraits(context.Background(), prometheus.QueryTraits{SubqueryFree: true})
set := cq.Select(ctx, false, &storage.SelectHints{Start: 1000, End: 2000, Step: 60_000}, testMatchers(t)...)
assert.False(t, set.Next())
require.NoError(t, set.Err())
statements := recorder.Statements()
require.Len(t, statements, 1)
assert.Contains(t, statements[0].Query, "IN (SELECT fingerprint FROM signoz_metrics.time_series_v4")
assert.Contains(t, statements[0].Query, "argMax(value, unix_milli)")
}

View File

@@ -0,0 +1,184 @@
package clickhouseprometheusv2
import (
"sort"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/tsdb/chunkenc"
"github.com/prometheus/prometheus/util/annotations"
)
// series is one time series with samples stored as parallel slices, ordered
// by timestamp. The compact layout avoids per-sample allocations and keeps
// iteration cache friendly.
type series struct {
lset labels.Labels
ts []int64
vs []float64
}
var _ storage.Series = (*series)(nil)
func (s *series) Labels() labels.Labels {
return s.lset
}
func (s *series) Iterator(it chunkenc.Iterator) chunkenc.Iterator {
if fit, ok := it.(*floatIterator); ok {
fit.reset(s)
return fit
}
fit := &floatIterator{}
fit.reset(s)
return fit
}
// floatIterator implements chunkenc.Iterator over a series' sample slices.
type floatIterator struct {
s *series
i int
}
var _ chunkenc.Iterator = (*floatIterator)(nil)
func (it *floatIterator) reset(s *series) {
it.s = s
it.i = -1
}
func (it *floatIterator) Next() chunkenc.ValueType {
it.i++
if it.i >= len(it.s.ts) {
return chunkenc.ValNone
}
return chunkenc.ValFloat
}
func (it *floatIterator) Seek(t int64) chunkenc.ValueType { //nolint:govet // stdmethods flags io.Seeker; this is chunkenc.Iterator's Seek
if it.i < 0 {
it.i = 0
}
if it.i >= len(it.s.ts) {
return chunkenc.ValNone
}
// The current position, once valid, must not move backwards.
if it.s.ts[it.i] >= t {
return chunkenc.ValFloat
}
it.i += sort.Search(len(it.s.ts)-it.i, func(j int) bool {
return it.s.ts[it.i+j] >= t
})
if it.i >= len(it.s.ts) {
return chunkenc.ValNone
}
return chunkenc.ValFloat
}
func (it *floatIterator) At() (int64, float64) {
return it.s.ts[it.i], it.s.vs[it.i]
}
func (it *floatIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
return 0, nil
}
func (it *floatIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
return 0, nil
}
func (it *floatIterator) AtT() int64 {
return it.s.ts[it.i]
}
// AtST returns the current start timestamp; not tracked by this storage.
func (it *floatIterator) AtST() int64 {
return 0
}
func (it *floatIterator) Err() error {
return nil
}
// seriesSet iterates a fully materialized, label-sorted list of series.
type seriesSet struct {
series []*series
i int
}
var _ storage.SeriesSet = (*seriesSet)(nil)
func newSeriesSet(list []*series) *seriesSet {
return &seriesSet{series: list, i: -1}
}
func (s *seriesSet) Next() bool {
s.i++
return s.i < len(s.series)
}
func (s *seriesSet) At() storage.Series {
return s.series[s.i]
}
func (s *seriesSet) Err() error {
return nil
}
func (s *seriesSet) Warnings() annotations.Annotations {
return nil
}
// sortAndMerge orders series by label set and merges series whose label sets
// are identical. Distinct fingerprints can carry identical label sets (e.g.
// series differing only in a non-label dimension); Prometheus storages never
// expose duplicate label sets to the engine, so merge their samples by
// timestamp, keeping the first sample on ties.
func sortAndMerge(list []*series) []*series {
if len(list) < 2 {
return list
}
sort.Slice(list, func(i, j int) bool {
return labels.Compare(list[i].lset, list[j].lset) < 0
})
out := list[:1]
for _, s := range list[1:] {
last := out[len(out)-1]
if labels.Compare(last.lset, s.lset) != 0 {
out = append(out, s)
continue
}
merged := mergeSamples(last, s)
out[len(out)-1] = merged
}
return out
}
func mergeSamples(a, b *series) *series {
ts := make([]int64, 0, len(a.ts)+len(b.ts))
vs := make([]float64, 0, len(a.ts)+len(b.ts))
i, j := 0, 0
for i < len(a.ts) && j < len(b.ts) {
switch {
case a.ts[i] < b.ts[j]:
ts = append(ts, a.ts[i])
vs = append(vs, a.vs[i])
i++
case a.ts[i] > b.ts[j]:
ts = append(ts, b.ts[j])
vs = append(vs, b.vs[j])
j++
default:
ts = append(ts, a.ts[i])
vs = append(vs, a.vs[i])
i++
j++
}
}
ts = append(ts, a.ts[i:]...)
vs = append(vs, a.vs[i:]...)
ts = append(ts, b.ts[j:]...)
vs = append(vs, b.vs[j:]...)
return &series{lset: a.lset, ts: ts, vs: vs}
}

View File

@@ -0,0 +1,201 @@
package clickhouseprometheusv2
import (
"fmt"
"strconv"
"strings"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/huandu/go-sqlbuilder"
"github.com/prometheus/prometheus/model/labels"
)
// inlineFingerprintsLimit is the largest matched-series count inlined into
// the samples query as literals. Literals engage the samples primary key and
// avoid a second series-table scan; past a few thousand the statement itself
// becomes the cost, and the shard-local subquery filter wins. Not
// configurable: the crossover depends on statement parsing, not on any
// property of a deployment an operator could know better.
const inlineFingerprintsLimit = 5_000
// buildSeriesQuery renders the series lookup: one row per matched fingerprint
// with its labels.
func buildSeriesQuery(start, end int64, matchers []*labels.Matcher) (string, []any, error) {
adjustedStart, table := timeSeriesTableFor(start, end)
sb := sqlbuilder.NewSelectBuilder()
sb.Select("fingerprint", "any(labels)")
sb.From(fmt.Sprintf("%s.%s", databaseName, table))
if err := applySeriesConditions(sb, adjustedStart, end, matchers); err != nil {
return "", nil, err
}
sb.GroupBy("fingerprint")
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
return query, args, nil
}
// buildSamplesQuery renders the samples fetch for the series selected by the
// series lookup. Small matched sets pass inlineFingerprints — sorted uint64
// literals that engage the samples primary key; nil means the set exceeded
// the inline limit, and the filter becomes a semi-join re-running the series
// predicates against the shard-local series table (complete by fingerprint
// co-locality, see localTimeSeriesTable; a GLOBAL broadcast of the matched
// set would ship it to every shard instead). metricNames narrows the
// primary-key scan; when the selector had no __name__ equality, the names
// observed on the matched series are used. A non-nil lastPerStep groups to
// one (the last) sample per step bucket.
func buildSamplesQuery(start, end int64, metricNames []string, inlineFingerprints []uint64, matchers []*labels.Matcher, lastPerStep *lastSamplePerStep) (string, []any, error) {
sb := sqlbuilder.NewSelectBuilder()
if lastPerStep != nil {
// Aliases must not shadow source columns: ClickHouse resolves aliases
// in WHERE too, and "max(unix_milli) AS unix_milli" would put an
// aggregate into the WHERE clause (error 184).
sb.Select("fingerprint", "max(unix_milli) AS ts", "argMax(value, unix_milli) AS val", "argMax(flags, unix_milli) AS fl")
} else {
sb.Select("fingerprint", "unix_milli", "value", "flags")
}
sb.From(fmt.Sprintf("%s.%s", databaseName, distributedSamplesV4))
switch len(metricNames) {
case 0:
// No name constraint derivable; correct but unable to use the
// metric_name primary-key prefix.
case 1:
sb.Where(sb.EQ("metric_name", metricNames[0]))
default:
sb.Where(sb.In("metric_name", sqlbuilder.List(metricNames)))
}
// temporality precedes metric_name in the samples primary key; the
// fingerprints already come from these temporalities, so this only helps
// granule pruning.
sb.Where("temporality IN ['Cumulative', 'Unspecified']")
if inlineFingerprints != nil {
sb.Where("fingerprint " + inlineFingerprintFilter(inlineFingerprints))
} else {
sub := sqlbuilder.NewSelectBuilder()
sub.Select("fingerprint")
adjustedStart, table := timeSeriesTableFor(start, end)
sub.From(fmt.Sprintf("%s.%s", databaseName, localTimeSeriesTable(table)))
if err := applySeriesConditions(sub, adjustedStart, end, matchers); err != nil {
return "", nil, err
}
sb.Where(sb.In("fingerprint", sub))
}
sb.Where(sb.GTE("unix_milli", start), sb.LTE("unix_milli", end))
if lastPerStep != nil {
sb.GroupBy("fingerprint")
if expr := lastPerStep.bucketExpr(); expr != "" {
sb.GroupBy(expr)
}
sb.OrderBy("fingerprint", "ts")
} else {
sb.OrderBy("fingerprint", "unix_milli")
}
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
return query, args, nil
}
// applySeriesConditions adds the WHERE conditions of a series table scan for
// the given matchers and window. __name__ matchers translate to the
// metric_name column (all four matcher types — the v1 client silently
// returned nothing for regex metric names); every other matcher translates
// to a JSONExtractString condition on the labels column. An equality matcher
// against "" matches series without the label, mirroring PromQL, because
// JSONExtractString returns "" for missing keys. Regexes are anchored:
// PromQL matchers match the whole value, while ClickHouse match() searches
// for a partial match — without anchoring, =~"api" would also select
// "x-api-y".
func applySeriesConditions(sb *sqlbuilder.SelectBuilder, start, end int64, matchers []*labels.Matcher) error {
for _, m := range matchers {
if m.Name != metricNameLabel {
continue
}
switch m.Type {
case labels.MatchEqual:
sb.Where(sb.EQ("metric_name", m.Value))
case labels.MatchNotEqual:
sb.Where(sb.NE("metric_name", m.Value))
case labels.MatchRegexp:
sb.Where(fmt.Sprintf("match(metric_name, %s)", sb.Var(anchorRegex(m.Value))))
case labels.MatchNotRegexp:
sb.Where(fmt.Sprintf("NOT match(metric_name, %s)", sb.Var(anchorRegex(m.Value))))
default:
return errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported matcher type %q for __name__", m.Type)
}
}
sb.Where("temporality IN ['Cumulative', 'Unspecified']")
// Inclusive upper bound: registration rows are hour-floored (and 6h/1d/1w
// for the rollup tables) by the exporter, so a series first registered in
// the bucket starting exactly at `end` would otherwise be invisible while
// its samples (<= end) are in range.
sb.Where(sb.GTE("unix_milli", start), sb.LTE("unix_milli", end))
for _, m := range matchers {
if m.Name == metricNameLabel {
continue
}
switch m.Type {
case labels.MatchEqual:
sb.Where(fmt.Sprintf("JSONExtractString(labels, %s) = %s", sb.Var(m.Name), sb.Var(m.Value)))
case labels.MatchNotEqual:
sb.Where(fmt.Sprintf("JSONExtractString(labels, %s) != %s", sb.Var(m.Name), sb.Var(m.Value)))
case labels.MatchRegexp:
sb.Where(fmt.Sprintf("match(JSONExtractString(labels, %s), %s)", sb.Var(m.Name), sb.Var(anchorRegex(m.Value))))
case labels.MatchNotRegexp:
sb.Where(fmt.Sprintf("NOT match(JSONExtractString(labels, %s), %s)", sb.Var(m.Name), sb.Var(anchorRegex(m.Value))))
default:
return errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported matcher type %q", m.Type)
}
}
return nil
}
// anchorRegex turns a PromQL regex into its fully-anchored form (see
// applySeriesConditions).
func anchorRegex(v string) string {
return "^(?:" + v + ")$"
}
// inlineFingerprintFilter renders "IN (fp1, fp2, ...)" with literal uint64s.
func inlineFingerprintFilter(fingerprints []uint64) string {
var b strings.Builder
b.Grow(len(fingerprints)*21 + 8)
b.WriteString("IN (")
for i, fp := range fingerprints {
if i > 0 {
b.WriteString(", ")
}
b.WriteString(strconv.FormatUint(fp, 10))
}
b.WriteString(")")
return b.String()
}
// lastSamplePerStep reduces an instant-selector fetch to the last sample of
// each step bucket. Buckets are anchored at the selector's first evaluation
// timestamp so that every bucket boundary coincides with an evaluation
// timestamp: bucket 0 is (start, firstEval] (the initial lookback window)
// and bucket i is (firstEval+(i-1)·step, firstEval+i·step]. Keeping only the
// last sample per bucket is lossless: the engine resolves each evaluation
// timestamp t to the latest sample in (t-lookback, t], and a non-final
// sample of a bucket can never be that latest sample for any t on the
// evaluation grid. Real timestamps are preserved, so the engine's own
// lookback and staleness handling remain exact.
type lastSamplePerStep struct {
firstEvalMs int64
stepMs int64
}
func (t *lastSamplePerStep) bucketExpr() string {
if t.stepMs <= 0 {
// Instant query: a single evaluation at firstEval; one bucket.
return ""
}
return fmt.Sprintf(
"if(unix_milli <= %d, 0, intDiv(unix_milli - %d - 1, %d) + 1)",
t.firstEvalMs, t.firstEvalMs, t.stepMs,
)
}

View File

@@ -0,0 +1,148 @@
package clickhouseprometheusv2
import (
"testing"
"time"
"github.com/prometheus/prometheus/model/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func mustMatcher(t *testing.T, mt labels.MatchType, name, value string) *labels.Matcher {
t.Helper()
m, err := labels.NewMatcher(mt, name, value)
require.NoError(t, err)
return m
}
func TestTimeSeriesTableFor(t *testing.T) {
base := time.Date(2026, 7, 10, 3, 27, 0, 0, time.UTC).UnixMilli()
tests := []struct {
name string
span time.Duration
wantTable string
roundTo time.Duration
}{
{"under 6h uses hourly table", 2 * time.Hour, distributedTimeSeriesV4, time.Hour},
{"under 1d uses 6h table", 12 * time.Hour, distributedTimeSeriesV46hrs, 6 * time.Hour},
{"under 1w uses 1d table", 3 * 24 * time.Hour, distributedTimeSeriesV41day, 24 * time.Hour},
{"over 1w uses 1w table", 10 * 24 * time.Hour, distributedTimeSeriesV41week, 7 * 24 * time.Hour},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
start, table := timeSeriesTableFor(base, base+tt.span.Milliseconds())
assert.Equal(t, tt.wantTable, table)
assert.Zero(t, start%tt.roundTo.Milliseconds())
assert.LessOrEqual(t, start, base)
})
}
}
func TestBuildSeriesQuery(t *testing.T) {
start := int64(1_700_000_000_000)
end := start + time.Hour.Milliseconds()
// The series table window rounds down to the table's bucket boundary.
adjustedStart := start - (start % time.Hour.Milliseconds())
t.Run("equality name and label matchers", func(t *testing.T) {
query, args, err := buildSeriesQuery(start, end, []*labels.Matcher{
mustMatcher(t, labels.MatchEqual, "__name__", "http_requests_total"),
mustMatcher(t, labels.MatchEqual, "job", "api"),
})
require.NoError(t, err)
assert.Equal(t,
"SELECT fingerprint, any(labels) FROM signoz_metrics.distributed_time_series_v4 WHERE metric_name = ? AND temporality IN ['Cumulative', 'Unspecified'] AND unix_milli >= ? AND unix_milli <= ? AND JSONExtractString(labels, ?) = ? GROUP BY fingerprint",
query,
)
assert.Equal(t, []any{"http_requests_total", adjustedStart, end, "job", "api"}, args)
})
t.Run("regex matchers are anchored", func(t *testing.T) {
_, args, err := buildSeriesQuery(start, end, []*labels.Matcher{
mustMatcher(t, labels.MatchEqual, "__name__", "up"),
mustMatcher(t, labels.MatchRegexp, "instance", "prod.*"),
mustMatcher(t, labels.MatchNotRegexp, "env", "dev|test"),
})
require.NoError(t, err)
assert.Equal(t, []any{"up", adjustedStart, end, "instance", "^(?:prod.*)$", "env", "^(?:dev|test)$"}, args)
})
t.Run("regex name matcher uses metric_name column", func(t *testing.T) {
query, args, err := buildSeriesQuery(start, end, []*labels.Matcher{
mustMatcher(t, labels.MatchRegexp, "__name__", "node_cpu.*|node_memory.*"),
})
require.NoError(t, err)
assert.Contains(t, query, "match(metric_name, ?)")
assert.NotContains(t, query, "JSONExtractString")
assert.Equal(t, []any{"^(?:node_cpu.*|node_memory.*)$", adjustedStart, end}, args)
})
t.Run("no name matcher omits metric_name condition", func(t *testing.T) {
query, _, err := buildSeriesQuery(start, end, []*labels.Matcher{
mustMatcher(t, labels.MatchEqual, "job", "api"),
})
require.NoError(t, err)
assert.NotContains(t, query, "metric_name")
})
}
func TestBuildSamplesQuery(t *testing.T) {
start := int64(1_700_000_000_000)
end := start + time.Hour.Milliseconds()
adjustedStart := start - (start % time.Hour.Milliseconds())
matchers := []*labels.Matcher{
mustMatcher(t, labels.MatchEqual, "__name__", "up"),
mustMatcher(t, labels.MatchEqual, "job", "api"),
}
t.Run("raw with inline fingerprints", func(t *testing.T) {
query, args, err := buildSamplesQuery(start, end, []string{"up"}, []uint64{7, 42}, matchers, nil)
require.NoError(t, err)
assert.Equal(t,
"SELECT fingerprint, unix_milli, value, flags FROM signoz_metrics.distributed_samples_v4 WHERE metric_name = ? AND temporality IN ['Cumulative', 'Unspecified'] AND fingerprint IN (7, 42) AND unix_milli >= ? AND unix_milli <= ? ORDER BY fingerprint, unix_milli",
query,
)
assert.Equal(t, []any{"up", start, end}, args)
})
t.Run("last-sample-per-step groups by step bucket anchored at first eval", func(t *testing.T) {
lastPerStep := &lastSamplePerStep{firstEvalMs: start + 299_999, stepMs: 60_000}
query, _, err := buildSamplesQuery(start, end, []string{"up"}, []uint64{7}, matchers, lastPerStep)
require.NoError(t, err)
assert.Contains(t, query, "argMax(value, unix_milli) AS val")
assert.Contains(t, query, "argMax(flags, unix_milli) AS fl")
assert.Contains(t, query, "GROUP BY fingerprint, if(unix_milli <= 1700000299999, 0, intDiv(unix_milli - 1700000299999 - 1, 60000) + 1)")
assert.Contains(t, query, "ORDER BY fingerprint, ts")
// Aliases must not shadow the source columns referenced in WHERE.
assert.NotContains(t, query, "AS unix_milli")
assert.NotContains(t, query, "AS value")
assert.NotContains(t, query, "AS flags")
})
t.Run("instant query keeps one bucket", func(t *testing.T) {
lastPerStep := &lastSamplePerStep{firstEvalMs: end, stepMs: 0}
query, _, err := buildSamplesQuery(start, end, []string{"up"}, []uint64{7}, matchers, lastPerStep)
require.NoError(t, err)
assert.Contains(t, query, "GROUP BY fingerprint ORDER BY fingerprint, ts")
assert.NotContains(t, query, "intDiv")
})
t.Run("over-limit set becomes a shard-local semi-join", func(t *testing.T) {
query, args, err := buildSamplesQuery(start, end, []string{"up"}, nil, matchers, nil)
require.NoError(t, err)
assert.Contains(t, query, "fingerprint IN (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE ")
assert.NotContains(t, query, "GLOBAL IN")
// Args follow placeholder order: samples metric name, the semi-join's
// series predicates, then the samples window bounds.
assert.Equal(t, []any{"up", "up", adjustedStart, end, "job", "api", start, end}, args)
})
t.Run("multiple metric names from regex selector", func(t *testing.T) {
query, args, err := buildSamplesQuery(start, end, []string{"node_cpu", "node_memory"}, []uint64{7}, matchers, nil)
require.NoError(t, err)
assert.Contains(t, query, "metric_name IN (?, ?)")
assert.Equal(t, []any{"node_cpu", "node_memory", start, end}, args)
})
}

View File

@@ -0,0 +1,65 @@
package clickhouseprometheusv2
import "time"
const (
// metricNameLabel is the reserved PromQL label holding the metric name.
metricNameLabel string = "__name__"
databaseName string = "signoz_metrics"
distributedTimeSeriesV4 string = "distributed_time_series_v4"
distributedTimeSeriesV46hrs string = "distributed_time_series_v4_6hrs"
distributedTimeSeriesV41day string = "distributed_time_series_v4_1day"
distributedTimeSeriesV41week string = "distributed_time_series_v4_1week"
distributedSamplesV4 string = "distributed_samples_v4"
localTimeSeriesV4 string = "time_series_v4"
localTimeSeriesV46hrs string = "time_series_v4_6hrs"
localTimeSeriesV41day string = "time_series_v4_1day"
localTimeSeriesV41week string = "time_series_v4_1week"
)
// localTimeSeriesTable maps a distributed time series table to its shard-local
// table. Samples and time series shard on the same key
// (cityHash64(env, temporality, metric_name, fingerprint)), so a query whose
// top-level FROM is the distributed samples table can join or semi-join the
// local time series table inside each shard: the shard rewrite runs the
// subquery against the shard's own series rows, which are exactly the series
// of the shard's samples. No broadcast, no initiator-side join.
func localTimeSeriesTable(distributed string) string {
switch distributed {
case distributedTimeSeriesV46hrs:
return localTimeSeriesV46hrs
case distributedTimeSeriesV41day:
return localTimeSeriesV41day
case distributedTimeSeriesV41week:
return localTimeSeriesV41week
default:
return localTimeSeriesV4
}
}
var (
oneHourInMilliseconds = time.Hour.Milliseconds()
sixHoursInMilliseconds = time.Hour.Milliseconds() * 6
oneDayInMilliseconds = time.Hour.Milliseconds() * 24
oneWeekInMilliseconds = time.Hour.Milliseconds() * 24 * 7
)
// timeSeriesTableFor returns the adjusted start and the time series table for
// the window. Time series tables hold one row per (fingerprint, bucket), with
// bucket granularities of 1h, 6h, 1d and 1w; the start is rounded down to the
// bucket boundary so a window beginning mid-bucket still matches the bucket's
// row.
func timeSeriesTableFor(start, end int64) (int64, string) {
switch {
case end-start < sixHoursInMilliseconds:
return start - (start % oneHourInMilliseconds), distributedTimeSeriesV4
case end-start < oneDayInMilliseconds:
return start - (start % sixHoursInMilliseconds), distributedTimeSeriesV46hrs
case end-start < oneWeekInMilliseconds:
return start - (start % oneDayInMilliseconds), distributedTimeSeriesV41day
default:
return start - (start % oneWeekInMilliseconds), distributedTimeSeriesV41week
}
}

View File

@@ -13,6 +13,11 @@ type ActiveQueryTrackerConfig struct {
MaxConcurrent int `mapstructure:"max_concurrent"`
}
type ClickhouseV2Config struct {
MaxFetchedSeries int `mapstructure:"max_fetched_series"`
MaxFetchedSamples int64 `mapstructure:"max_fetched_samples"`
}
type Config struct {
ActiveQueryTrackerConfig ActiveQueryTrackerConfig `mapstructure:"active_query_tracker"`
@@ -24,6 +29,9 @@ type Config struct {
// Timeout is the maximum time a query is allowed to run before being aborted.
Timeout time.Duration `mapstructure:"timeout"`
// ClickhouseV2 configures the clickhousev2 provider.
ClickhouseV2 ClickhouseV2Config `mapstructure:"clickhousev2"`
}
func NewConfigFactory() factory.ConfigFactory {
@@ -38,6 +46,10 @@ func newConfig() factory.Config {
MaxConcurrent: 20,
},
Timeout: 2 * time.Minute,
ClickhouseV2: ClickhouseV2Config{
MaxFetchedSeries: 500_000,
MaxFetchedSamples: 50_000_000,
},
}
}
@@ -45,6 +57,9 @@ func (c Config) Validate() error {
if c.Timeout <= 0 {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "prometheus::timeout must be greater than 0")
}
if c.ClickhouseV2.MaxFetchedSeries < 0 || c.ClickhouseV2.MaxFetchedSamples < 0 {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "prometheus::clickhousev2 limits must not be negative")
}
return nil
}

49
pkg/prometheus/traits.go Normal file
View File

@@ -0,0 +1,49 @@
package prometheus
import (
"context"
"github.com/prometheus/prometheus/promql/parser"
)
type queryTraitsKey struct{}
// QueryTraits carries per-query facts a storage implementation cannot derive
// from SelectHints alone. Call sites that parse the PromQL expression attach
// traits to the context before handing it to the engine; storages treat a
// missing traits value as "unknown" and stay conservative.
type QueryTraits struct {
// SubqueryFree is true when the query contains no subquery expression.
// Subquery selectors are evaluated at the subquery's own step, but
// SelectHints.Step always carries the top-level step, so step-aligned
// storage optimizations (e.g. keeping only the last sample per step
// bucket) are safe only when this is true.
SubqueryFree bool
}
// DetectQueryTraits derives QueryTraits from a parsed PromQL expression.
func DetectQueryTraits(expr parser.Expr) QueryTraits {
subqueryFree := true
parser.Inspect(expr, func(node parser.Node, _ []parser.Node) error {
if _, ok := node.(*parser.SubqueryExpr); ok {
subqueryFree = false
}
return nil
})
return QueryTraits{SubqueryFree: subqueryFree}
}
// NewContextWithQueryTraits returns a context carrying the given traits.
func NewContextWithQueryTraits(ctx context.Context, traits QueryTraits) context.Context {
return context.WithValue(ctx, queryTraitsKey{}, traits)
}
// QueryTraitsFromContext returns the traits attached to ctx, if any.
//
// Context is used here, unlike for backend selection, because traits must
// cross the promql engine to reach storage.Querier.Select, and the engine's
// interfaces offer no other channel; the alternative is a Prometheus fork.
func QueryTraitsFromContext(ctx context.Context) (QueryTraits, bool) {
traits, ok := ctx.Value(queryTraitsKey{}).(QueryTraits)
return traits, ok
}

View File

@@ -22,13 +22,6 @@ import (
"github.com/SigNoz/signoz/pkg/variables"
)
type Handler interface {
QueryRange(rw http.ResponseWriter, req *http.Request)
QueryRangePreview(rw http.ResponseWriter, req *http.Request)
QueryRawStream(rw http.ResponseWriter, req *http.Request)
ReplaceVariables(rw http.ResponseWriter, req *http.Request)
}
type handler struct {
set factory.ProviderSettings
analytics analytics.Analytics

View File

@@ -15,14 +15,6 @@ import (
"github.com/SigNoz/signoz/pkg/valuer"
)
// BucketCache is the interface for bucket-based caching.
type BucketCache interface {
// cached portion + list of gaps to fetch
GetMissRanges(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step) (cached *qbtypes.Result, missing []*qbtypes.TimeRange)
// store fresh buckets for future hits
Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result)
}
// bucketCache implements the BucketCache interface.
type bucketCache struct {
cache cache.Cache

View File

@@ -11,8 +11,8 @@ import (
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrytraces"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
@@ -277,12 +277,12 @@ func (q *builderQuery[T]) narrowWindowByTraceID(ctx context.Context, fromMS, toM
return fromMS, toMS, true, ""
}
traceIDs, found := tracestelemetryschema.ExtractTraceIDsFromFilter(q.spec.Filter.Expression)
traceIDs, found := telemetrytraces.ExtractTraceIDsFromFilter(q.spec.Filter.Expression)
if !found || len(traceIDs) == 0 {
return fromMS, toMS, true, ""
}
finder := tracestelemetryschema.NewTraceTimeRangeFinder(q.telemetryStore)
finder := telemetrytraces.NewTraceTimeRangeFinder(q.telemetryStore)
traceStart, traceEnd, exists, err := finder.GetTraceTimeRangeMulti(ctx, traceIDs)
if err != nil {
return fromMS, toMS, true, ""

View File

@@ -6,18 +6,18 @@ import (
"time"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/telemetrytraces"
"github.com/SigNoz/signoz/pkg/valuer"
)
func (q *querier) Collect(ctx context.Context, _ valuer.UUID) (map[string]any, error) {
stats := make(map[string]any)
tracesTable := fmt.Sprintf("%s.%s", tracestelemetryschema.DBName, tracestelemetryschema.SpanIndexV3TableName)
logsTable := fmt.Sprintf("%s.%s", logstelemetryschema.DBName, logstelemetryschema.LogsV2TableName)
metricsTable := fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4TableName)
tracesTable := fmt.Sprintf("%s.%s", telemetrytraces.DBName, telemetrytraces.SpanIndexV3TableName)
logsTable := fmt.Sprintf("%s.%s", telemetrylogs.DBName, telemetrylogs.LogsV2TableName)
metricsTable := fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4TableName)
var (
traces uint64

View File

@@ -9,6 +9,12 @@ import (
const DefaultMaxConcurrentQueries = 8
type SkipResourceFingerprint struct {
Enabled bool `yaml:"enabled" mapstructure:"enabled"`
// If count of fingerprint is above threshold, skip the fingerprint subquery and filter on main table instead.
Threshold uint64 `yaml:"threshold" mapstructure:"threshold"`
}
// Config represents the configuration for the querier.
type Config struct {
// CacheTTL is the TTL for cached query results
@@ -17,6 +23,8 @@ type Config struct {
FluxInterval time.Duration `yaml:"flux_interval" mapstructure:"flux_interval"`
// MaxConcurrentQueries is the maximum number of concurrent queries for missing ranges
MaxConcurrentQueries int `yaml:"max_concurrent_queries" mapstructure:"max_concurrent_queries"`
// SkipResourceFingerprint configures when the resource fingerprint subquery is skipped in favor of main-table filtering.
SkipResourceFingerprint SkipResourceFingerprint `yaml:"skip_resource_fingerprint" mapstructure:"skip_resource_fingerprint"`
// LogTraceIDWindowPadding is the padding added to narrowed down timerange from trace summary to logs with trace_id filter.
LogTraceIDWindowPadding time.Duration `yaml:"log_trace_id_window_padding" mapstructure:"log_trace_id_window_padding"`
}
@@ -31,7 +39,11 @@ func newConfig() factory.Config {
// Default values
CacheTTL: 168 * time.Hour,
FluxInterval: 5 * time.Minute,
MaxConcurrentQueries: DefaultMaxConcurrentQueries,
MaxConcurrentQueries: DefaultMaxConcurrentQueries,
SkipResourceFingerprint: SkipResourceFingerprint{
Enabled: false,
Threshold: 100000,
},
LogTraceIDWindowPadding: 5 * time.Minute,
}
}
@@ -47,6 +59,9 @@ func (c Config) Validate() error {
if c.MaxConcurrentQueries <= 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "max_concurrent_queries must be positive, got %v", c.MaxConcurrentQueries)
}
if c.SkipResourceFingerprint.Enabled && c.SkipResourceFingerprint.Threshold == 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "skip_resource_fingerprint.threshold must be > 0 when enabled")
}
if c.LogTraceIDWindowPadding < 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "log_trace_id_window_padding must not be negative, got %v", c.LogTraceIDWindowPadding)
}

35
pkg/querier/interfaces.go Normal file
View File

@@ -0,0 +1,35 @@
package querier
import (
"context"
"net/http"
"github.com/SigNoz/signoz/pkg/statsreporter"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/valuer"
)
// Querier interface defines the contract for querying data.
type Querier interface {
QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (*qbtypes.QueryRangeResponse, error)
QueryRawStream(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, client *qbtypes.RawStream)
statsreporter.StatsCollector
// QueryRangePreview validates and renders the queries without executing them.
QueryRangePreview(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, opts qbtypes.QueryRangePreviewOptions) (*qbtypes.QueryRangePreviewResponse, error)
}
// BucketCache is the interface for bucket-based caching.
type BucketCache interface {
// cached portion + list of gaps to fetch
GetMissRanges(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step) (cached *qbtypes.Result, missing []*qbtypes.TimeRange)
// store fresh buckets for future hits
Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result)
}
type Handler interface {
QueryRange(rw http.ResponseWriter, req *http.Request)
// QueryRangePreview is the dry-run endpoint: validate and render without executing.
QueryRangePreview(rw http.ResponseWriter, req *http.Request)
QueryRawStream(rw http.ResponseWriter, req *http.Request)
ReplaceVariables(rw http.ResponseWriter, req *http.Request)
}

View File

@@ -331,11 +331,14 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
return nil, errors.WrapInternalf(promErr, errors.CodeInternal, "error getting matrix from promql query %q", query)
}
// Hide only known SigNoz storage keys: label names are user data and may
// legitimately start with "__" (e.g. __address__), so a blanket dunder
// strip mangles user labelsets. The __scope./__resource. prefixes cover
// every exporter version's keys.
excludeLabel := func(labelName string) bool {
if labelName == "__name__" {
return false
}
return strings.HasPrefix(labelName, "__") || labelName == "fingerprint"
return labelName == "__temporality__" ||
strings.HasPrefix(labelName, "__scope.") ||
strings.HasPrefix(labelName, "__resource.")
}
var series []*qbv5.TimeSeries

View File

@@ -21,8 +21,6 @@ import (
"github.com/SigNoz/signoz/pkg/prometheus"
"github.com/SigNoz/signoz/pkg/query-service/utils"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder"
"github.com/SigNoz/signoz/pkg/statsreporter"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
@@ -37,15 +35,6 @@ var (
intervalWarn = "Query %s is requesting aggregation interval %v seconds, which is smaller than the minimum allowed interval of %v seconds for selected time range. Using the minimum instead"
)
// Querier interface defines the contract for querying data.
type Querier interface {
QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (*qbtypes.QueryRangeResponse, error)
QueryRawStream(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, client *qbtypes.RawStream)
statsreporter.StatsCollector
// QueryRangePreview validates and renders the queries without executing them.
QueryRangePreview(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, opts qbtypes.QueryRangePreviewOptions) (*qbtypes.QueryRangePreviewResponse, error)
}
type querier struct {
logger *slog.Logger
fl flagger.Flagger
@@ -71,7 +60,12 @@ func New(
telemetryStore telemetrystore.TelemetryStore,
metadataStore telemetrytypes.MetadataStore,
promEngine prometheus.Prometheus,
builders *statementbuilder.Builders,
traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation],
meterStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation],
traceOperatorStmtBuilder qbtypes.TraceOperatorStatementBuilder,
bucketCache BucketCache,
flagger flagger.Flagger,
logTraceIDWindowPadding time.Duration,
@@ -81,21 +75,18 @@ func New(
if maxConcurrentQueries <= 0 {
maxConcurrentQueries = DefaultMaxConcurrentQueries
}
if builders == nil {
builders = &statementbuilder.Builders{}
}
return &querier{
logger: querierSettings.Logger(),
fl: flagger,
telemetryStore: telemetryStore,
metadataStore: metadataStore,
promEngine: promEngine,
traceStmtBuilder: builders.Trace,
logStmtBuilder: builders.Log,
auditStmtBuilder: builders.Audit,
metricStmtBuilder: builders.Metric,
meterStmtBuilder: builders.Meter,
traceOperatorStmtBuilder: builders.TraceOperator,
traceStmtBuilder: traceStmtBuilder,
logStmtBuilder: logStmtBuilder,
auditStmtBuilder: auditStmtBuilder,
metricStmtBuilder: metricStmtBuilder,
meterStmtBuilder: meterStmtBuilder,
traceOperatorStmtBuilder: traceOperatorStmtBuilder,
bucketCache: bucketCache,
liveDataRefresh: 5 * time.Second,
builderConfig: builderConfig{

View File

@@ -11,7 +11,6 @@ import (
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/statementbuilder"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
@@ -49,7 +48,12 @@ func TestQueryRange_MetricTypeMissing(t *testing.T) {
nil, // telemetryStore
metadataStore,
nil, // prometheus
nil, // builders
nil, // traceStmtBuilder
nil, // logStmtBuilder
nil, // auditStmtBuilder
nil, // metricStmtBuilder
nil, // meterStmtBuilder
nil, // traceOperatorStmtBuilder
nil, // bucketCache
flaggertest.New(t), // flagger
0, // logTraceIDWindowPadding
@@ -115,12 +119,17 @@ func TestQueryRange_MetricTypeFromStore(t *testing.T) {
providerSettings,
telemetryStore,
metadataStore,
nil, // prometheus
&statementbuilder.Builders{Metric: &mockMetricStmtBuilder{}},
nil, // bucketCache
flaggertest.New(t), // flagger
0, // logTraceIDWindowPadding
0, // maxConcurrentQueries
nil, // prometheus
nil, // traceStmtBuilder
nil, // logStmtBuilder
nil, // auditStmtBuilder
&mockMetricStmtBuilder{}, // metricStmtBuilder
nil, // meterStmtBuilder
nil, // traceOperatorStmtBuilder
nil, // bucketCache
flaggertest.New(t), // flagger
0, // logTraceIDWindowPadding
0, // maxConcurrentQueries
)
req := &qbtypes.QueryRangeRequest{

View File

@@ -1,52 +0,0 @@
package queriertest
import (
"context"
"github.com/SigNoz/signoz/pkg/querier"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/valuer"
)
// MockQuerier implements querier.Querier for testing. Each behavior can be
// overridden via a function field; unset fields return empty, non-error values.
type MockQuerier struct {
QueryRangeFunc func(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (*qbtypes.QueryRangeResponse, error)
QueryRawStreamFunc func(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, client *qbtypes.RawStream)
QueryRangePreviewFunc func(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, opts qbtypes.QueryRangePreviewOptions) (*qbtypes.QueryRangePreviewResponse, error)
CollectFunc func(ctx context.Context, orgID valuer.UUID) (map[string]any, error)
}
var _ querier.Querier = (*MockQuerier)(nil)
// NewMockQuerier creates a new MockQuerier.
func NewMockQuerier() *MockQuerier {
return &MockQuerier{}
}
func (m *MockQuerier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (*qbtypes.QueryRangeResponse, error) {
if m.QueryRangeFunc != nil {
return m.QueryRangeFunc(ctx, orgID, req)
}
return &qbtypes.QueryRangeResponse{}, nil
}
func (m *MockQuerier) QueryRawStream(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, client *qbtypes.RawStream) {
if m.QueryRawStreamFunc != nil {
m.QueryRawStreamFunc(ctx, orgID, req, client)
}
}
func (m *MockQuerier) QueryRangePreview(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, opts qbtypes.QueryRangePreviewOptions) (*qbtypes.QueryRangePreviewResponse, error) {
if m.QueryRangePreviewFunc != nil {
return m.QueryRangePreviewFunc(ctx, orgID, req, opts)
}
return &qbtypes.QueryRangePreviewResponse{}, nil
}
func (m *MockQuerier) Collect(ctx context.Context, orgID valuer.UUID) (map[string]any, error) {
if m.CollectFunc != nil {
return m.CollectFunc(ctx, orgID)
}
return map[string]any{}, nil
}

View File

@@ -3,44 +3,192 @@ package signozquerier
import (
"context"
"github.com/SigNoz/signoz/pkg/cache"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/prometheus"
"github.com/SigNoz/signoz/pkg/querier"
"github.com/SigNoz/signoz/pkg/statementbuilder"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryaudit"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/telemetrymetadata"
"github.com/SigNoz/signoz/pkg/telemetrymeter"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/telemetrytraces"
)
// NewFactory creates a new factory for the signoz querier provider. The query
// stack (metadata store, statement builders, bucket cache) is assembled once in
// signoz.go and injected here.
// NewFactory creates a new factory for the signoz querier provider.
func NewFactory(
telemetryStore telemetrystore.TelemetryStore,
prometheus prometheus.Prometheus,
metadataStore telemetrytypes.MetadataStore,
builders *statementbuilder.Builders,
bucketCache querier.BucketCache,
cache cache.Cache,
flagger flagger.Flagger,
) factory.ProviderFactory[querier.Querier, querier.Config] {
return factory.NewProviderFactory(
factory.MustNewName("signoz"),
func(
_ context.Context,
ctx context.Context,
settings factory.ProviderSettings,
cfg querier.Config,
) (querier.Querier, error) {
return querier.New(
settings,
telemetryStore,
metadataStore,
prometheus,
builders,
bucketCache,
flagger,
cfg.LogTraceIDWindowPadding,
cfg.MaxConcurrentQueries,
), nil
return newProvider(ctx, settings, cfg, telemetryStore, prometheus, cache, flagger)
},
)
}
func newProvider(
_ context.Context,
settings factory.ProviderSettings,
cfg querier.Config,
telemetryStore telemetrystore.TelemetryStore,
prometheus prometheus.Prometheus,
cache cache.Cache,
flagger flagger.Flagger,
) (querier.Querier, error) {
// Create telemetry metadata store
telemetryMetadataStore := telemetrymetadata.NewTelemetryMetaStore(
settings,
telemetryStore,
telemetrytraces.DBName,
telemetrytraces.TagAttributesV2TableName,
telemetrytraces.SpanAttributesKeysTblName,
telemetrytraces.SpanIndexV3TableName,
telemetrymetrics.DBName,
telemetrymetrics.AttributesMetadataTableName,
telemetrymeter.DBName,
telemetrymeter.SamplesAgg1dTableName,
telemetrylogs.DBName,
telemetrylogs.LogsV2TableName,
telemetrylogs.TagAttributesV2TableName,
telemetrylogs.LogAttributeKeysTblName,
telemetrylogs.LogResourceKeysTblName,
telemetryaudit.DBName,
telemetryaudit.AuditLogsTableName,
telemetryaudit.TagAttributesTableName,
telemetryaudit.LogAttributeKeysTblName,
telemetryaudit.LogResourceKeysTblName,
telemetrymetadata.DBName,
telemetrymetadata.AttributesMetadataTableName,
telemetrymetadata.ColumnEvolutionMetadataTableName,
flagger,
)
// Create trace statement builder
traceFieldMapper := telemetrytraces.NewFieldMapper()
traceConditionBuilder := telemetrytraces.NewConditionBuilder(traceFieldMapper)
traceAggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, traceFieldMapper, traceConditionBuilder, flagger)
traceStmtBuilder := telemetrytraces.NewTraceQueryStatementBuilder(
settings,
telemetryMetadataStore,
traceFieldMapper,
traceConditionBuilder,
traceAggExprRewriter,
telemetryStore,
flagger,
cfg.SkipResourceFingerprint.Enabled,
cfg.SkipResourceFingerprint.Threshold,
)
// Create trace operator statement builder
traceOperatorStmtBuilder := telemetrytraces.NewTraceOperatorStatementBuilder(
settings,
telemetryMetadataStore,
traceFieldMapper,
traceConditionBuilder,
traceStmtBuilder,
traceAggExprRewriter,
flagger,
)
// Create log statement builder
logFieldMapper := telemetrylogs.NewFieldMapper(flagger)
logConditionBuilder := telemetrylogs.NewConditionBuilder(logFieldMapper, flagger)
logAggExprRewriter := querybuilder.NewAggExprRewriter(
settings,
telemetrylogs.DefaultFullTextColumn,
logFieldMapper,
logConditionBuilder,
flagger,
)
logStmtBuilder := telemetrylogs.NewLogQueryStatementBuilder(
settings,
telemetryMetadataStore,
logFieldMapper,
logConditionBuilder,
logAggExprRewriter,
telemetrylogs.DefaultFullTextColumn,
flagger,
telemetryStore,
cfg.SkipResourceFingerprint.Enabled,
cfg.SkipResourceFingerprint.Threshold,
)
// Create audit statement builder
auditFieldMapper := telemetryaudit.NewFieldMapper()
auditConditionBuilder := telemetryaudit.NewConditionBuilder(auditFieldMapper)
auditAggExprRewriter := querybuilder.NewAggExprRewriter(
settings,
telemetryaudit.DefaultFullTextColumn,
auditFieldMapper,
auditConditionBuilder,
flagger,
)
auditStmtBuilder := telemetryaudit.NewAuditQueryStatementBuilder(
settings,
telemetryMetadataStore,
auditFieldMapper,
auditConditionBuilder,
auditAggExprRewriter,
telemetryaudit.DefaultFullTextColumn,
flagger,
)
// Create metric statement builder
metricFieldMapper := telemetrymetrics.NewFieldMapper()
metricConditionBuilder := telemetrymetrics.NewConditionBuilder(metricFieldMapper)
metricStmtBuilder := telemetrymetrics.NewMetricQueryStatementBuilder(
settings,
telemetryMetadataStore,
metricFieldMapper,
metricConditionBuilder,
flagger,
)
// Create meter statement builder
meterStmtBuilder := telemetrymeter.NewMeterQueryStatementBuilder(
settings,
telemetryMetadataStore,
metricFieldMapper,
metricConditionBuilder,
metricStmtBuilder,
)
// Create bucket cache
bucketCache := querier.NewBucketCache(
settings,
cache,
cfg.CacheTTL,
cfg.FluxInterval,
)
// Create and return the querier
return querier.New(
settings,
telemetryStore,
telemetryMetadataStore,
prometheus,
traceStmtBuilder,
logStmtBuilder,
auditStmtBuilder,
metricStmtBuilder,
meterStmtBuilder,
traceOperatorStmtBuilder,
bucketCache,
flagger,
cfg.LogTraceIDWindowPadding,
cfg.MaxConcurrentQueries,
), nil
}

View File

@@ -18,13 +18,6 @@ import (
"github.com/SigNoz/signoz/pkg/querier/signozquerier"
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/SigNoz/signoz/pkg/sqlstore/sqlstoretest"
"github.com/SigNoz/signoz/pkg/statementbuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/auditstatementbuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/logsstatementbuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/meterstatementbuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/tracesstatementbuilder"
"github.com/SigNoz/signoz/pkg/telemetrymetadata"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
"github.com/stretchr/testify/require"
@@ -112,31 +105,7 @@ func NewTestManager(t *testing.T, testOpts *TestManagerOptions) *Manager {
}
// Create querier with test values
metadataStore := telemetrymetadata.NewTelemetryMetaStore(providerSettings, telemetryStore, flagger)
cfg := statementbuilder.Config{}
ctx := context.Background()
traceStmtBuilder, err := tracesstatementbuilder.NewFactory(telemetryStore, metadataStore, flagger).New(ctx, providerSettings, cfg)
require.NoError(t, err)
traceOperatorStmtBuilder, err := tracesstatementbuilder.NewOperatorFactory(telemetryStore, metadataStore, flagger).New(ctx, providerSettings, cfg)
require.NoError(t, err)
logStmtBuilder, err := logsstatementbuilder.NewFactory(telemetryStore, metadataStore, flagger).New(ctx, providerSettings, cfg)
require.NoError(t, err)
auditStmtBuilder, err := auditstatementbuilder.NewFactory(metadataStore, flagger).New(ctx, providerSettings, cfg)
require.NoError(t, err)
metricStmtBuilder, err := metricsstatementbuilder.NewFactory(metadataStore, flagger).New(ctx, providerSettings, cfg)
require.NoError(t, err)
meterStmtBuilder, err := meterstatementbuilder.NewFactory(metadataStore, flagger).New(ctx, providerSettings, cfg)
require.NoError(t, err)
builders := &statementbuilder.Builders{
Trace: traceStmtBuilder,
TraceOperator: traceOperatorStmtBuilder,
Log: logStmtBuilder,
Audit: auditStmtBuilder,
Metric: metricStmtBuilder,
Meter: meterStmtBuilder,
}
bucketCache := querier.NewBucketCache(providerSettings, cache, 0, 0)
providerFactory := signozquerier.NewFactory(telemetryStore, prometheus, metadataStore, builders, bucketCache, flagger)
providerFactory := signozquerier.NewFactory(telemetryStore, prometheus, cache, flagger)
mockQuerier, err := providerFactory.New(context.Background(), providerSettings, querier.Config{})
require.NoError(t, err)

View File

@@ -8,11 +8,11 @@ import (
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/querier"
"github.com/SigNoz/signoz/pkg/statementbuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/logsstatementbuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/tracesstatementbuilder"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrytraces"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
"github.com/stretchr/testify/require"
@@ -24,7 +24,7 @@ func prepareQuerierForMetrics(t *testing.T, telemetryStore telemetrystore.Teleme
providerSettings := instrumentationtest.New().ToProviderSettings()
metadataStore := telemetrytypestest.NewMockMetadataStore()
fl, err := flagger.New(
flagger, err := flagger.New(
context.Background(),
instrumentationtest.New().ToProviderSettings(),
flagger.Config{},
@@ -32,17 +32,29 @@ func prepareQuerierForMetrics(t *testing.T, telemetryStore telemetrystore.Teleme
)
require.NoError(t, err)
metricStmtBuilder, err := metricsstatementbuilder.NewFactory(metadataStore, fl).New(context.Background(), providerSettings, statementbuilder.Config{})
require.NoError(t, err)
metricFieldMapper := telemetrymetrics.NewFieldMapper()
metricConditionBuilder := telemetrymetrics.NewConditionBuilder(metricFieldMapper)
metricStmtBuilder := telemetrymetrics.NewMetricQueryStatementBuilder(
providerSettings,
metadataStore,
metricFieldMapper,
metricConditionBuilder,
flagger,
)
return querier.New(
providerSettings,
telemetryStore,
metadataStore,
nil, // prometheus
&statementbuilder.Builders{Metric: metricStmtBuilder},
nil, // traceStmtBuilder
nil, // logStmtBuilder
nil, // auditStmtBuilder
metricStmtBuilder,
nil, // meterStmtBuilder
nil, // traceOperatorStmtBuilder
nil, // bucketCache
fl,
flagger,
0,
0, // maxConcurrentQueries (0 means default)
), metadataStore
@@ -61,16 +73,40 @@ func prepareQuerierForLogs(t *testing.T, telemetryStore telemetrystore.Telemetry
metadataStore.KeysMap = keysMap
fl := flaggertest.New(t)
logStmtBuilder, err := logsstatementbuilder.NewFactory(nil, metadataStore, fl).New(context.Background(), providerSettings, statementbuilder.Config{})
require.NoError(t, err)
logFieldMapper := telemetrylogs.NewFieldMapper(fl)
logConditionBuilder := telemetrylogs.NewConditionBuilder(logFieldMapper, fl)
logAggExprRewriter := querybuilder.NewAggExprRewriter(
providerSettings,
telemetrylogs.DefaultFullTextColumn,
logFieldMapper,
logConditionBuilder,
fl,
)
logStmtBuilder := telemetrylogs.NewLogQueryStatementBuilder(
providerSettings,
metadataStore,
logFieldMapper,
logConditionBuilder,
logAggExprRewriter,
telemetrylogs.DefaultFullTextColumn,
fl,
nil,
false,
100000,
)
return querier.New(
providerSettings,
telemetryStore,
metadataStore,
nil, // prometheus
&statementbuilder.Builders{Log: logStmtBuilder},
nil, // bucketCache
nil, // prometheus
nil, // traceStmtBuilder
logStmtBuilder, // logStmtBuilder
nil, // auditStmtBuilder
nil, // metricStmtBuilder
nil, // meterStmtBuilder
nil, // traceOperatorStmtBuilder
nil, // bucketCache
fl,
5*time.Minute, // logTraceIDWindowPadding
0, // maxConcurrentQueries (0 means default)
@@ -90,17 +126,36 @@ func prepareQuerierForTraces(t *testing.T, telemetryStore telemetrystore.Telemet
}
metadataStore.KeysMap = keysMap
// Create trace statement builder
traceFieldMapper := telemetrytraces.NewFieldMapper()
traceConditionBuilder := telemetrytraces.NewConditionBuilder(traceFieldMapper)
fl := flaggertest.New(t)
traceStmtBuilder, err := tracesstatementbuilder.NewFactory(telemetryStore, metadataStore, fl).New(context.Background(), providerSettings, statementbuilder.Config{})
require.NoError(t, err)
traceAggExprRewriter := querybuilder.NewAggExprRewriter(providerSettings, nil, traceFieldMapper, traceConditionBuilder, fl)
traceStmtBuilder := telemetrytraces.NewTraceQueryStatementBuilder(
providerSettings,
metadataStore,
traceFieldMapper,
traceConditionBuilder,
traceAggExprRewriter,
telemetryStore,
fl,
false,
100000,
)
return querier.New(
providerSettings,
telemetryStore,
metadataStore,
nil, // prometheus
&statementbuilder.Builders{Trace: traceStmtBuilder},
nil, // bucketCache
nil, // prometheus
traceStmtBuilder, // traceStmtBuilder
nil, // logStmtBuilder
nil, // auditStmtBuilder
nil, // metricStmtBuilder
nil, // meterStmtBuilder
nil, // traceOperatorStmtBuilder
nil, // bucketCache
fl,
0,
0, // maxConcurrentQueries (0 means default)

View File

@@ -39,7 +39,6 @@ import (
"github.com/SigNoz/signoz/pkg/sqlmigrator"
"github.com/SigNoz/signoz/pkg/sqlschema"
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/SigNoz/signoz/pkg/statementbuilder"
"github.com/SigNoz/signoz/pkg/statsreporter"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/tokenizer"
@@ -98,9 +97,6 @@ type Config struct {
// Querier config
Querier querier.Config `mapstructure:"querier"`
// StatementBuilder config
StatementBuilder statementbuilder.Config `mapstructure:"statementbuilder"`
// Ruler config
Ruler ruler.Config `mapstructure:"ruler"`
@@ -170,7 +166,6 @@ func NewConfig(ctx context.Context, logger *slog.Logger, resolverConfig config.R
prometheus.NewConfigFactory(),
alertmanager.NewConfigFactory(),
querier.NewConfigFactory(),
statementbuilder.NewConfigFactory(),
ruler.NewConfigFactory(),
emailing.NewConfigFactory(),
sharder.NewConfigFactory(),

View File

@@ -56,7 +56,6 @@ import (
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/SigNoz/signoz/pkg/sqlstore/sqlitesqlstore"
"github.com/SigNoz/signoz/pkg/sqlstore/sqlstorehook"
"github.com/SigNoz/signoz/pkg/statementbuilder"
"github.com/SigNoz/signoz/pkg/statsreporter"
"github.com/SigNoz/signoz/pkg/statsreporter/analyticsstatsreporter"
"github.com/SigNoz/signoz/pkg/statsreporter/noopstatsreporter"
@@ -70,7 +69,6 @@ import (
"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
"github.com/SigNoz/signoz/pkg/types/featuretypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/version"
"github.com/SigNoz/signoz/pkg/web"
"github.com/SigNoz/signoz/pkg/web/noopweb"
@@ -287,9 +285,9 @@ func NewStatsReporterProviderFactories(aggregator statsreporter.Aggregator, orgG
)
}
func NewQuerierProviderFactories(telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, metadataStore telemetrytypes.MetadataStore, builders *statementbuilder.Builders, bucketCache querier.BucketCache, flagger flagger.Flagger) factory.NamedMap[factory.ProviderFactory[querier.Querier, querier.Config]] {
func NewQuerierProviderFactories(telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, cache cache.Cache, flagger flagger.Flagger) factory.NamedMap[factory.ProviderFactory[querier.Querier, querier.Config]] {
return factory.MustNewNamedMap(
signozquerier.NewFactory(telemetryStore, prometheus, metadataStore, builders, bucketCache, flagger),
signozquerier.NewFactory(telemetryStore, prometheus, cache, flagger),
)
}

View File

@@ -48,15 +48,14 @@ import (
"github.com/SigNoz/signoz/pkg/sqlmigrator"
"github.com/SigNoz/signoz/pkg/sqlschema"
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/SigNoz/signoz/pkg/statementbuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/auditstatementbuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/logsstatementbuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/meterstatementbuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/tracesstatementbuilder"
"github.com/SigNoz/signoz/pkg/statsreporter"
"github.com/SigNoz/signoz/pkg/telemetryaudit"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/telemetrymetadata"
"github.com/SigNoz/signoz/pkg/telemetrymeter"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrytraces"
pkgtokenizer "github.com/SigNoz/signoz/pkg/tokenizer"
"github.com/SigNoz/signoz/pkg/types/authtypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -97,61 +96,6 @@ type SigNoz struct {
MeterReporter meterreporter.Reporter
}
// newQueryStack assembles the query stack once: the shared telemetry metadata
// store, the per-signal statement builders, and the bucket cache. The metadata
// store is returned so callers can reuse the single instance downstream.
func newQueryStack(
ctx context.Context,
settings factory.ProviderSettings,
config Config,
telemetryStore telemetrystore.TelemetryStore,
cache cache.Cache,
fl flagger.Flagger,
) (telemetrytypes.MetadataStore, *statementbuilder.Builders, querier.BucketCache, error) {
metadataStore := telemetrymetadata.NewTelemetryMetaStore(settings, telemetryStore, fl)
// Composition root: build each per-signal statement-builder factory and assemble
// the Builders bundle. This is the only place that imports the concrete sub-packages.
cfg := config.StatementBuilder
traceStmtBuilder, err := tracesstatementbuilder.NewFactory(telemetryStore, metadataStore, fl).New(ctx, settings, cfg)
if err != nil {
return nil, nil, nil, err
}
traceOperatorStmtBuilder, err := tracesstatementbuilder.NewOperatorFactory(telemetryStore, metadataStore, fl).New(ctx, settings, cfg)
if err != nil {
return nil, nil, nil, err
}
logStmtBuilder, err := logsstatementbuilder.NewFactory(telemetryStore, metadataStore, fl).New(ctx, settings, cfg)
if err != nil {
return nil, nil, nil, err
}
auditStmtBuilder, err := auditstatementbuilder.NewFactory(metadataStore, fl).New(ctx, settings, cfg)
if err != nil {
return nil, nil, nil, err
}
metricStmtBuilder, err := metricsstatementbuilder.NewFactory(metadataStore, fl).New(ctx, settings, cfg)
if err != nil {
return nil, nil, nil, err
}
meterStmtBuilder, err := meterstatementbuilder.NewFactory(metadataStore, fl).New(ctx, settings, cfg)
if err != nil {
return nil, nil, nil, err
}
builders := &statementbuilder.Builders{
Trace: traceStmtBuilder,
TraceOperator: traceOperatorStmtBuilder,
Log: logStmtBuilder,
Audit: auditStmtBuilder,
Metric: metricStmtBuilder,
Meter: meterStmtBuilder,
}
bucketCache := querier.NewBucketCache(settings, cache, config.Querier.CacheTTL, config.Querier.FluxInterval)
return metadataStore, builders, bucketCache, nil
}
func New(
ctx context.Context,
config Config,
@@ -310,19 +254,12 @@ func New(
return nil, err
}
// Assemble the query stack (metadata store, statement builders, bucket cache) once,
// and reuse the single metadata store everywhere downstream.
telemetryMetadataStore, builders, bucketCache, err := newQueryStack(ctx, providerSettings, config, telemetrystore, cache, flagger)
if err != nil {
return nil, err
}
// Initialize querier from the available querier provider factories
querier, err := factory.NewProviderFromNamedMap(
ctx,
providerSettings,
config.Querier,
NewQuerierProviderFactories(telemetrystore, prometheus, telemetryMetadataStore, builders, bucketCache, flagger),
NewQuerierProviderFactories(telemetrystore, prometheus, cache, flagger),
config.Querier.Provider(),
)
if err != nil {
@@ -490,6 +427,35 @@ func New(
return nil, err
}
// Initialize telemetry metadata store
// TODO: consolidate other telemetrymetadata.NewTelemetryMetaStore initializations to reuse this instance instead.
telemetryMetadataStore := telemetrymetadata.NewTelemetryMetaStore(
providerSettings,
telemetrystore,
telemetrytraces.DBName,
telemetrytraces.TagAttributesV2TableName,
telemetrytraces.SpanAttributesKeysTblName,
telemetrytraces.SpanIndexV3TableName,
telemetrymetrics.DBName,
telemetrymetrics.AttributesMetadataTableName,
telemetrymeter.DBName,
telemetrymeter.SamplesAgg1dTableName,
telemetrylogs.DBName,
telemetrylogs.LogsV2TableName,
telemetrylogs.TagAttributesV2TableName,
telemetrylogs.LogAttributeKeysTblName,
telemetrylogs.LogResourceKeysTblName,
telemetryaudit.DBName,
telemetryaudit.AuditLogsTableName,
telemetryaudit.TagAttributesTableName,
telemetryaudit.LogAttributeKeysTblName,
telemetryaudit.LogResourceKeysTblName,
telemetrymetadata.DBName,
telemetrymetadata.AttributesMetadataTableName,
telemetrymetadata.ColumnEvolutionMetadataTableName,
flagger,
)
global, err := factory.NewProviderFromNamedMap(
ctx,
providerSettings,

View File

@@ -1,32 +0,0 @@
package auditstatementbuilder
import (
"context"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
// NewFactory returns a provider factory for the audit statement builder. Its New
// internalizes the FieldMapper, ConditionBuilder, and AggExprRewriter.
func NewFactory(
metadataStore telemetrytypes.MetadataStore,
fl flagger.Flagger,
) factory.ProviderFactory[qbtypes.StatementBuilder[qbtypes.LogAggregation], statementbuilder.Config] {
return factory.NewProviderFactory(
factory.MustNewName("audit"),
func(_ context.Context, settings factory.ProviderSettings, _ statementbuilder.Config) (qbtypes.StatementBuilder[qbtypes.LogAggregation], error) {
fm := audittelemetryschema.NewFieldMapper()
cb := audittelemetryschema.NewConditionBuilder(fm)
aggExprRewriter := querybuilder.NewAggExprRewriter(settings, audittelemetryschema.DefaultFullTextColumn, fm, cb, fl)
return NewAuditQueryStatementBuilder(
settings, metadataStore, fm, cb, aggExprRewriter, audittelemetryschema.DefaultFullTextColumn, fl,
), nil
},
)
}

View File

@@ -1,41 +0,0 @@
package statementbuilder
import (
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
)
// SkipResourceFingerprint configures when the resource fingerprint subquery is skipped in favor of main-table filtering.
type SkipResourceFingerprint struct {
Enabled bool `yaml:"enabled" mapstructure:"enabled"`
// If count of fingerprint is above threshold, skip the fingerprint subquery and filter on main table instead.
Threshold uint64 `yaml:"threshold" mapstructure:"threshold"`
}
// Config represents the configuration for the statement builders.
type Config struct {
// SkipResourceFingerprint configures when the resource fingerprint subquery is skipped in favor of main-table filtering.
SkipResourceFingerprint SkipResourceFingerprint `yaml:"skip_resource_fingerprint" mapstructure:"skip_resource_fingerprint"`
}
// NewConfigFactory creates a new config factory for the statement builders.
func NewConfigFactory() factory.ConfigFactory {
return factory.NewConfigFactory(factory.MustNewName("statementbuilder"), newConfig)
}
func newConfig() factory.Config {
return Config{
SkipResourceFingerprint: SkipResourceFingerprint{
Enabled: false,
Threshold: 100000,
},
}
}
// Validate validates the configuration.
func (c Config) Validate() error {
if c.SkipResourceFingerprint.Enabled && c.SkipResourceFingerprint.Threshold == 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "skip_resource_fingerprint.threshold must be > 0 when enabled")
}
return nil
}

View File

@@ -1,36 +0,0 @@
package logsstatementbuilder
import (
"context"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrystore"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
// NewFactory returns a provider factory for the logs statement builder. Its New
// internalizes the FieldMapper, ConditionBuilder, and AggExprRewriter, and reads
// SkipResourceFingerprint from the config.
func NewFactory(
telemetryStore telemetrystore.TelemetryStore,
metadataStore telemetrytypes.MetadataStore,
fl flagger.Flagger,
) factory.ProviderFactory[qbtypes.StatementBuilder[qbtypes.LogAggregation], statementbuilder.Config] {
return factory.NewProviderFactory(
factory.MustNewName("logs"),
func(_ context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.StatementBuilder[qbtypes.LogAggregation], error) {
fm := logstelemetryschema.NewFieldMapper(fl)
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
aggExprRewriter := querybuilder.NewAggExprRewriter(settings, logstelemetryschema.DefaultFullTextColumn, fm, cb, fl)
return NewLogQueryStatementBuilder(
settings, metadataStore, fm, cb, aggExprRewriter, logstelemetryschema.DefaultFullTextColumn,
fl, telemetryStore, cfg.SkipResourceFingerprint.Enabled, cfg.SkipResourceFingerprint.Threshold,
), nil
},
)
}

View File

@@ -1,34 +0,0 @@
package meterstatementbuilder
import (
"context"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/statementbuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
// NewFactory returns a provider factory for the meter statement builder. Its New
// reuses the metrics FieldMapper/ConditionBuilder and delegates the final SELECT
// to a metrics statement builder built via the metrics factory.
func NewFactory(
metadataStore telemetrytypes.MetadataStore,
fl flagger.Flagger,
) factory.ProviderFactory[qbtypes.StatementBuilder[qbtypes.MetricAggregation], statementbuilder.Config] {
return factory.NewProviderFactory(
factory.MustNewName("meter"),
func(ctx context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.StatementBuilder[qbtypes.MetricAggregation], error) {
metricsStatementBuilder, err := metricsstatementbuilder.NewFactory(metadataStore, fl).New(ctx, settings, cfg)
if err != nil {
return nil, err
}
fm := metricstelemetryschema.NewFieldMapper()
cb := metricstelemetryschema.NewConditionBuilder(fm)
return NewMeterQueryStatementBuilder(settings, metadataStore, fm, cb, metricsStatementBuilder), nil
},
)
}

View File

@@ -1,28 +0,0 @@
package metricsstatementbuilder
import (
"context"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/statementbuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
// NewFactory returns a provider factory for the metrics statement builder. Its
// New internalizes the FieldMapper and ConditionBuilder and yields the concrete
// *StatementBuilder so the meter builder can reuse it.
func NewFactory(
metadataStore telemetrytypes.MetadataStore,
fl flagger.Flagger,
) factory.ProviderFactory[*StatementBuilder, statementbuilder.Config] {
return factory.NewProviderFactory(
factory.MustNewName("metrics"),
func(_ context.Context, settings factory.ProviderSettings, _ statementbuilder.Config) (*StatementBuilder, error) {
fm := metricstelemetryschema.NewFieldMapper()
cb := metricstelemetryschema.NewConditionBuilder(fm)
return NewMetricQueryStatementBuilder(settings, metadataStore, fm, cb, fl), nil
},
)
}

View File

@@ -1,22 +0,0 @@
package statementbuilder
import (
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
)
// Builders bundles the per-signal statement builders that make up the query stack.
//
// This package is contract-only: it declares Builders (here) and Config (config.go)
// and imports no sub-package. Each <signal>statementbuilder depends on this package
// for Config and exposes its own factory.ProviderFactory[..., Config]; the composition
// root (signoz.go::newQueryStack) is the only place that imports the concrete
// sub-packages and assembles this struct. Keeping the edge as subs -> parent (never
// the reverse) is what lets the sub-packages reference Config without an import cycle.
type Builders struct {
Trace qbtypes.StatementBuilder[qbtypes.TraceAggregation]
TraceOperator qbtypes.TraceOperatorStatementBuilder
Log qbtypes.StatementBuilder[qbtypes.LogAggregation]
Audit qbtypes.StatementBuilder[qbtypes.LogAggregation]
Metric qbtypes.StatementBuilder[qbtypes.MetricAggregation]
Meter qbtypes.StatementBuilder[qbtypes.MetricAggregation]
}

View File

@@ -1,62 +0,0 @@
package tracesstatementbuilder
import (
"context"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrystore"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
// NewFactory returns a provider factory for the trace query statement builder. Its
// New internalizes the FieldMapper, ConditionBuilder, and AggExprRewriter, and reads
// SkipResourceFingerprint from the config.
func NewFactory(
telemetryStore telemetrystore.TelemetryStore,
metadataStore telemetrytypes.MetadataStore,
fl flagger.Flagger,
) factory.ProviderFactory[qbtypes.StatementBuilder[qbtypes.TraceAggregation], statementbuilder.Config] {
return factory.NewProviderFactory(
factory.MustNewName("traces"),
func(_ context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.StatementBuilder[qbtypes.TraceAggregation], error) {
fm := tracestelemetryschema.NewFieldMapper()
cb := tracestelemetryschema.NewConditionBuilder(fm)
aggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, fm, cb, fl)
return NewTraceQueryStatementBuilder(
settings, metadataStore, fm, cb, aggExprRewriter, telemetryStore, fl,
cfg.SkipResourceFingerprint.Enabled, cfg.SkipResourceFingerprint.Threshold,
), nil
},
)
}
// NewOperatorFactory returns a provider factory for the trace-operator statement
// builder. The operator delegates sub-query construction to a trace query statement
// builder, so it builds its own internally — mirroring how the meter factory builds
// its own metrics builder.
func NewOperatorFactory(
telemetryStore telemetrystore.TelemetryStore,
metadataStore telemetrytypes.MetadataStore,
fl flagger.Flagger,
) factory.ProviderFactory[qbtypes.TraceOperatorStatementBuilder, statementbuilder.Config] {
return factory.NewProviderFactory(
factory.MustNewName("traceoperator"),
func(_ context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.TraceOperatorStatementBuilder, error) {
fm := tracestelemetryschema.NewFieldMapper()
cb := tracestelemetryschema.NewConditionBuilder(fm)
aggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, fm, cb, fl)
traceStmtBuilder := NewTraceQueryStatementBuilder(
settings, metadataStore, fm, cb, aggExprRewriter, telemetryStore, fl,
cfg.SkipResourceFingerprint.Enabled, cfg.SkipResourceFingerprint.Threshold,
)
return NewTraceOperatorStatementBuilder(
settings, metadataStore, fm, cb, traceStmtBuilder, aggExprRewriter, fl,
), nil
},
)
}

View File

@@ -1,4 +1,4 @@
package audittelemetryschema
package telemetryaudit
import (
"context"

View File

@@ -1,4 +1,4 @@
package audittelemetryschema
package telemetryaudit
import (
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
@@ -127,14 +127,3 @@ var auditLogColumns = map[string]*schema.Column{
"scope_version": {Name: "scope_version", Type: schema.ColumnTypeString},
"scope_string": {Name: "scope_string", Type: schema.MapColumnType{KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, ValueType: schema.ColumnTypeString}},
}
const (
DBName = "signoz_audit"
AuditLogsTableName = "distributed_logs"
AuditLogsLocalTableName = "logs"
TagAttributesTableName = "distributed_tag_attributes"
TagAttributesLocalTableName = "tag_attributes"
LogAttributeKeysTblName = "distributed_logs_attribute_keys"
LogResourceKeysTblName = "distributed_logs_resource_keys"
LogsResourceTableName = "distributed_logs_resource"
)

View File

@@ -1,4 +1,4 @@
package audittelemetryschema
package telemetryaudit
import (
"context"

View File

@@ -1,4 +1,4 @@
package auditstatementbuilder
package telemetryaudit
import (
"context"
@@ -10,8 +10,7 @@ import (
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/resourcefilter"
"github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryresourcefilter"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
@@ -39,12 +38,12 @@ func NewAuditQueryStatementBuilder(
fullTextColumn *telemetrytypes.TelemetryFieldKey,
flagger flagger.Flagger,
) *auditQueryStatementBuilder {
auditSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema")
auditSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryaudit")
resourceFilterStmtBuilder := resourcefilter.New[qbtypes.LogAggregation](
resourceFilterStmtBuilder := telemetryresourcefilter.New[qbtypes.LogAggregation](
settings,
audittelemetryschema.DBName,
audittelemetryschema.LogsResourceTableName,
DBName,
LogsResourceTableName,
telemetrytypes.SignalLogs,
telemetrytypes.SourceAudit,
metadataStore,
@@ -192,8 +191,8 @@ func (b *auditQueryStatementBuilder) adjustKeys(ctx context.Context, keys map[st
}
func (b *auditQueryStatementBuilder) adjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) []string {
if _, ok := audittelemetryschema.IntrinsicFields[key.Name]; ok {
intrinsicField := audittelemetryschema.IntrinsicFields[key.Name]
if _, ok := IntrinsicFields[key.Name]; ok {
intrinsicField := IntrinsicFields[key.Name]
return querybuilder.AdjustKey(key, keys, &intrinsicField)
}
return querybuilder.AdjustKey(key, keys, nil)
@@ -220,26 +219,26 @@ func (b *auditQueryStatementBuilder) buildListQuery(
cteArgs = append(cteArgs, args)
}
sb.Select(audittelemetryschema.TimestampColumn)
sb.SelectMore(audittelemetryschema.IDColumn)
sb.Select(TimestampColumn)
sb.SelectMore(IDColumn)
if len(query.SelectFields) == 0 {
sb.SelectMore(audittelemetryschema.TraceIDColumn)
sb.SelectMore(audittelemetryschema.SpanIDColumn)
sb.SelectMore(audittelemetryschema.TraceFlagsColumn)
sb.SelectMore(audittelemetryschema.SeverityTextColumn)
sb.SelectMore(audittelemetryschema.SeverityNumberColumn)
sb.SelectMore(audittelemetryschema.ScopeNameColumn)
sb.SelectMore(audittelemetryschema.ScopeVersionColumn)
sb.SelectMore(audittelemetryschema.BodyColumn)
sb.SelectMore(audittelemetryschema.EventNameColumn)
sb.SelectMore(audittelemetryschema.AttributesStringColumn)
sb.SelectMore(audittelemetryschema.AttributesNumberColumn)
sb.SelectMore(audittelemetryschema.AttributesBoolColumn)
sb.SelectMore(audittelemetryschema.ResourceColumn)
sb.SelectMore(audittelemetryschema.ScopeStringColumn)
sb.SelectMore(TraceIDColumn)
sb.SelectMore(SpanIDColumn)
sb.SelectMore(TraceFlagsColumn)
sb.SelectMore(SeverityTextColumn)
sb.SelectMore(SeverityNumberColumn)
sb.SelectMore(ScopeNameColumn)
sb.SelectMore(ScopeVersionColumn)
sb.SelectMore(BodyColumn)
sb.SelectMore(EventNameColumn)
sb.SelectMore(AttributesStringColumn)
sb.SelectMore(AttributesNumberColumn)
sb.SelectMore(AttributesBoolColumn)
sb.SelectMore(ResourceColumn)
sb.SelectMore(ScopeStringColumn)
} else {
for index := range query.SelectFields {
if query.SelectFields[index].Name == audittelemetryschema.TimestampColumn || query.SelectFields[index].Name == audittelemetryschema.IDColumn {
if query.SelectFields[index].Name == TimestampColumn || query.SelectFields[index].Name == IDColumn {
continue
}
@@ -251,7 +250,7 @@ func (b *auditQueryStatementBuilder) buildListQuery(
}
}
sb.From(fmt.Sprintf("%s.%s", audittelemetryschema.DBName, audittelemetryschema.AuditLogsTableName))
sb.From(fmt.Sprintf("%s.%s", DBName, AuditLogsTableName))
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables)
if err != nil {
@@ -341,7 +340,7 @@ func (b *auditQueryStatementBuilder) buildTimeSeriesQuery(
sb.SelectMore(fmt.Sprintf("%s AS __result_%d", rewritten, i))
}
sb.From(fmt.Sprintf("%s.%s", audittelemetryschema.DBName, audittelemetryschema.AuditLogsTableName))
sb.From(fmt.Sprintf("%s.%s", DBName, AuditLogsTableName))
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables)
if err != nil {
@@ -479,7 +478,7 @@ func (b *auditQueryStatementBuilder) buildScalarQuery(
}
}
sb.From(fmt.Sprintf("%s.%s", audittelemetryschema.DBName, audittelemetryschema.AuditLogsTableName))
sb.From(fmt.Sprintf("%s.%s", DBName, AuditLogsTableName))
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables)
if err != nil {

View File

@@ -1,4 +1,4 @@
package auditstatementbuilder
package telemetryaudit
import (
"context"
@@ -8,7 +8,6 @@ import (
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
@@ -55,8 +54,8 @@ func newTestAuditStatementBuilder(t *testing.T) *auditQueryStatementBuilder {
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = auditFieldKeyMap()
fm := audittelemetryschema.NewFieldMapper()
cb := audittelemetryschema.NewConditionBuilder(fm)
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
return NewAuditQueryStatementBuilder(
@@ -65,7 +64,7 @@ func newTestAuditStatementBuilder(t *testing.T) *auditQueryStatementBuilder {
fm,
cb,
aggExprRewriter,
audittelemetryschema.DefaultFullTextColumn,
DefaultFullTextColumn,
fl,
)
}

View File

@@ -0,0 +1,12 @@
package telemetryaudit
const (
DBName = "signoz_audit"
AuditLogsTableName = "distributed_logs"
AuditLogsLocalTableName = "logs"
TagAttributesTableName = "distributed_tag_attributes"
TagAttributesLocalTableName = "tag_attributes"
LogAttributeKeysTblName = "distributed_logs_attribute_keys"
LogResourceKeysTblName = "distributed_logs_resource_keys"
LogsResourceTableName = "distributed_logs_resource"
)

View File

@@ -1,4 +1,4 @@
package logstelemetryschema
package telemetrylogs
import (
"context"

View File

@@ -1,4 +1,4 @@
package logstelemetryschema
package telemetrylogs
import (
"context"
@@ -548,7 +548,7 @@ func TestConditionForSynthesizedPrefixedKeys(t *testing.T) {
t.Run("bare intrinsic column resolves to the column, not synthesized attributes", func(t *testing.T) {
sb := sqlbuilder.NewSelectBuilder()
key := telemetrytypes.TelemetryFieldKey{Name: "severity_text"}
conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "ERROR", sb)
conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "ERROR", sb)
require.NoError(t, err)
require.Len(t, conds, 1)
sb.Where(conds...)
@@ -560,7 +560,7 @@ func TestConditionForSynthesizedPrefixedKeys(t *testing.T) {
t.Run("attribute context", func(t *testing.T) {
sb := sqlbuilder.NewSelectBuilder()
key := telemetrytypes.TelemetryFieldKey{Name: "custom.key", FieldContext: telemetrytypes.FieldContextAttribute}
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "v", sb)
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "v", sb)
require.NoError(t, err)
assert.NotEmpty(t, warnings)
require.Len(t, conds, 2)
@@ -571,7 +571,7 @@ func TestConditionForSynthesizedPrefixedKeys(t *testing.T) {
t.Run("resource context", func(t *testing.T) {
sb := sqlbuilder.NewSelectBuilder()
key := telemetrytypes.TelemetryFieldKey{Name: "custom.key", FieldContext: telemetrytypes.FieldContextResource}
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "v", sb)
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "v", sb)
require.NoError(t, err)
assert.NotEmpty(t, warnings)
require.Len(t, conds, 2)
@@ -582,7 +582,7 @@ func TestConditionForSynthesizedPrefixedKeys(t *testing.T) {
t.Run("log context folds to attributes then body", func(t *testing.T) {
sb := sqlbuilder.NewSelectBuilder()
key := telemetrytypes.TelemetryFieldKey{Name: "custom.key", FieldContext: telemetrytypes.FieldContextLog}
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "v", sb)
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "v", sb)
require.NoError(t, err)
assert.NotEmpty(t, warnings)
require.Len(t, conds, 4)

View File

@@ -1,6 +1,8 @@
package logstelemetryschema
package telemetrylogs
import (
"fmt"
"github.com/SigNoz/signoz-otel-collector/constants"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -39,8 +41,9 @@ const (
// messageSubColumn is the ClickHouse sub-column that body searches map to
// when use_json_body feature flag is true.
messageSubField = "message"
messageSubColumn = "body_v2.message"
messageSubField = "message"
messageSubColumn = "body_v2.message"
bodySearchDefaultWarning = "body searches default to `body.message:string`. Use `body.<key>` to search a different field inside body"
// bodyMessageField is the field name addressing the message sub-field of the
// body when use_json_body is enabled (i.e. `body` + `.` + `message`). hasToken
@@ -134,16 +137,10 @@ var (
}
)
const (
DBName = "signoz_logs"
LogsV2TableName = "distributed_logs_v2"
LogsV2LocalTableName = "logs_v2"
TagAttributesV2TableName = "distributed_tag_attributes_v2"
TagAttributesV2LocalTableName = "tag_attributes_v2"
LogAttributeKeysTblName = "distributed_logs_attribute_keys"
LogResourceKeysTblName = "distributed_logs_resource_keys"
LogsResourceV2TableName = "distributed_logs_v2_resource"
PathTypesTableName = "distributed_json_path_types"
PromotedPathsTableName = "distributed_json_promoted_paths"
SkipIndexTableName = "system.data_skipping_indices"
)
func bodyAliasExpression(bodyJSONEnabled bool) string {
if !bodyJSONEnabled {
return LogsV2BodyColumn
}
return fmt.Sprintf("%s as body", LogsV2BodyV2Column)
}

View File

@@ -1,4 +1,4 @@
package logstelemetryschema
package telemetrylogs
import (
"context"

View File

@@ -1,4 +1,4 @@
package logstelemetryschema
package telemetrylogs
import (
"context"

View File

@@ -1,4 +1,4 @@
package logstelemetryschema
package telemetrylogs
import (
"context"
@@ -19,7 +19,7 @@ func TestLikeAndILikeWithoutWildcards_Warns(t *testing.T) {
cb := NewConditionBuilder(fm, fl)
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
keys := BuildCompleteFieldKeyMap(releaseTime)
keys := buildCompleteFieldKeyMap(releaseTime)
opts := querybuilder.FilterExprVisitorOpts{
Context: ctx,
@@ -57,7 +57,7 @@ func TestLikeAndILikeWithWildcards_NoWarn(t *testing.T) {
cb := NewConditionBuilder(fm, fl)
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
keys := BuildCompleteFieldKeyMap(releaseTime)
keys := buildCompleteFieldKeyMap(releaseTime)
opts := querybuilder.FilterExprVisitorOpts{
Context: context.Background(),

View File

@@ -1,4 +1,4 @@
package logstelemetryschema
package telemetrylogs
import (
"context"
@@ -21,7 +21,7 @@ func TestFilterExprLogsBodyJSON(t *testing.T) {
cb := NewConditionBuilder(fm, fl)
// Define a comprehensive set of field keys to support all test cases
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
keys := BuildCompleteFieldKeyMap(releaseTime)
keys := buildCompleteFieldKeyMap(releaseTime)
opts := querybuilder.FilterExprVisitorOpts{
Context: context.Background(),

View File

@@ -1,4 +1,4 @@
package logstelemetryschema
package telemetrylogs
import (
"context"
@@ -41,7 +41,7 @@ func TestFilterExprLogs(t *testing.T) {
cb := NewConditionBuilder(fm, fl)
// Define a comprehensive set of field keys to support all test cases
keys := BuildCompleteFieldKeyMap(releaseTime)
keys := buildCompleteFieldKeyMap(releaseTime)
opts := querybuilder.FilterExprVisitorOpts{
Context: ctx,
@@ -2476,7 +2476,7 @@ func TestFilterExprLogsConflictNegation(t *testing.T) {
// Define a comprehensive set of field keys to support all test cases
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
keys := BuildCompleteFieldKeyMap(releaseTime)
keys := buildCompleteFieldKeyMap(releaseTime)
keys["body"] = []*telemetrytypes.TelemetryFieldKey{
{

View File

@@ -1,4 +1,4 @@
package logsstatementbuilder
package telemetrylogs
import (
"context"
@@ -8,7 +8,6 @@ import (
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
@@ -44,14 +43,14 @@ func TestStatementBuilderGroupByUnknownKey(t *testing.T) {
t.Run(c.name, func(t *testing.T) {
fl := flaggertest.WithUseJSONBody(t, c.useJSONBody)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
fm := logstelemetryschema.NewFieldMapper(fl)
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
fm := NewFieldMapper(fl)
cb := NewConditionBuilder(fm, fl)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
statementBuilder := NewLogQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore, fm, cb, aggExprRewriter,
logstelemetryschema.DefaultFullTextColumn, fl, nil, false, 100000,
DefaultFullTextColumn, fl, nil, false, 100000,
)
query := qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{

View File

@@ -1,4 +1,4 @@
package logstelemetryschema
package telemetrylogs
import (
"fmt"

View File

@@ -1,4 +1,4 @@
package logstelemetryschema
package telemetrylogs
import (
"strings"

View File

@@ -1,4 +1,4 @@
package logsstatementbuilder
package telemetrylogs
import (
"context"
@@ -11,7 +11,6 @@ import (
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
@@ -1226,7 +1225,7 @@ func TestJSONStmtBuilder_OrderBy(t *testing.T) {
func TestResourceAggrAndGroupBy_WithJSONEnabled(t *testing.T) {
statementBuilder, metadataStore := buildJSONTestStatementBuilder(t, false)
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
keysMap := logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
keysMap := buildCompleteFieldKeyMap(releaseTime)
for _, keys := range keysMap {
for _, key := range keys {
metadataStore.SetKey(key)
@@ -1288,7 +1287,7 @@ func TestResourceAggrAndGroupBy_WithJSONEnabled(t *testing.T) {
func buildTestTelemetryMetadataStore(t *testing.T, addIndexes bool) *telemetrytypestest.MockMetadataStore {
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.SetStaticFields(logstelemetryschema.IntrinsicFields)
mockMetadataStore.SetStaticFields(IntrinsicFields)
types, _ := telemetrytypes.TestJSONTypeSet()
for path, fieldDataTypes := range types {
for _, fdt := range fieldDataTypes {
@@ -1308,15 +1307,15 @@ func buildTestTelemetryMetadataStore(t *testing.T, addIndexes bool) *telemetryty
Name: path,
FieldContext: telemetrytypes.FieldContextBody,
FieldDataType: fdt,
BaseColumn: logstelemetryschema.LogsV2BodyV2Column,
IndexExpression: schemamigrator.JSONSubColumnIndexExpr(logstelemetryschema.LogsV2BodyV2Column, path, jsonType.StringValue()),
BaseColumn: LogsV2BodyV2Column,
IndexExpression: schemamigrator.JSONSubColumnIndexExpr(LogsV2BodyV2Column, path, jsonType.StringValue()),
})
}
}
err := key.SetJSONAccessPlan(telemetrytypes.JSONColumnMetadata{
BaseColumn: logstelemetryschema.LogsV2BodyV2Column,
PromotedColumn: logstelemetryschema.LogsV2BodyPromotedColumn,
BaseColumn: LogsV2BodyV2Column,
PromotedColumn: LogsV2BodyPromotedColumn,
}, types)
require.NoError(t, err)
@@ -1332,8 +1331,8 @@ func buildJSONTestStatementBuilder(t *testing.T, addIndexes bool) (*logQueryStat
mockMetadataStore := buildTestTelemetryMetadataStore(t, addIndexes)
fl := flaggertest.WithUseJSONBody(t, true)
fm := logstelemetryschema.NewFieldMapper(fl)
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
fm := NewFieldMapper(fl)
cb := NewConditionBuilder(fm, fl)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
@@ -1343,7 +1342,7 @@ func buildJSONTestStatementBuilder(t *testing.T, addIndexes bool) (*logQueryStat
fm,
cb,
aggExprRewriter,
logstelemetryschema.DefaultFullTextColumn,
DefaultFullTextColumn,
fl,
nil,
false,

View File

@@ -1,4 +1,4 @@
package logstelemetryschema
package telemetrylogs
import (
"context"

View File

@@ -1,4 +1,4 @@
package logsstatementbuilder
package telemetrylogs
import (
"context"
@@ -10,8 +10,7 @@ import (
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/resourcefilter"
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryresourcefilter"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/featuretypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
@@ -20,22 +19,12 @@ import (
"github.com/huandu/go-sqlbuilder"
)
const bodySearchDefaultWarning = "body searches default to `body.message:string`. Use `body.<key>` to search a different field inside body"
func bodyAliasExpression(bodyJSONEnabled bool) string {
if !bodyJSONEnabled {
return logstelemetryschema.LogsV2BodyColumn
}
return fmt.Sprintf("%s as body", logstelemetryschema.LogsV2BodyV2Column)
}
type logQueryStatementBuilder struct {
logger *slog.Logger
metadataStore telemetrytypes.MetadataStore
fm qbtypes.FieldMapper
cb qbtypes.ConditionBuilder
resourceFilterResolver *resourcefilter.ResourceFingerprintResolver[qbtypes.LogAggregation]
resourceFilterResolver *telemetryresourcefilter.ResourceFingerprintResolver[qbtypes.LogAggregation]
aggExprRewriter qbtypes.AggExprRewriter
fl flagger.Flagger
skipResourceFingerprintEnabled bool
@@ -57,12 +46,12 @@ func NewLogQueryStatementBuilder(
skipResourceFingerprintEnable bool,
skipResourceFingerprintThreshold uint64,
) *logQueryStatementBuilder {
logsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema")
logsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrylogs")
resourceFilterResolver := resourcefilter.NewResolver[qbtypes.LogAggregation](
resourceFilterResolver := telemetryresourcefilter.NewResolver[qbtypes.LogAggregation](
settings,
logstelemetryschema.DBName,
logstelemetryschema.LogsResourceV2TableName,
DBName,
LogsResourceV2TableName,
telemetrytypes.SignalLogs,
telemetrytypes.SourceUnspecified,
metadataStore,
@@ -185,7 +174,7 @@ func getKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation], bo
// TODO(Piyush): Setup better for coming FTS support.
if bodyJSONEnabled {
for _, sel := range keySelectors {
if sel.Name == logstelemetryschema.LogsV2BodyColumn {
if sel.Name == LogsV2BodyColumn {
warnings = append(warnings, bodySearchDefaultWarning)
break
}
@@ -261,8 +250,8 @@ func (b *logQueryStatementBuilder) adjustKeys(ctx context.Context, keys map[stri
func (b *logQueryStatementBuilder) adjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) []string {
// First check if it matches with any intrinsic fields
var intrinsicOrCalculatedField telemetrytypes.TelemetryFieldKey
if _, ok := logstelemetryschema.IntrinsicFields[key.Name]; ok {
intrinsicOrCalculatedField = logstelemetryschema.IntrinsicFields[key.Name]
if _, ok := IntrinsicFields[key.Name]; ok {
intrinsicOrCalculatedField = IntrinsicFields[key.Name]
return querybuilder.AdjustKey(key, keys, &intrinsicOrCalculatedField)
}
@@ -295,28 +284,28 @@ func (b *logQueryStatementBuilder) buildListQuery(
}
// Select timestamp and id by default
sb.Select(logstelemetryschema.LogsV2TimestampColumn)
sb.SelectMore(logstelemetryschema.LogsV2IDColumn)
sb.Select(LogsV2TimestampColumn)
sb.SelectMore(LogsV2IDColumn)
if len(query.SelectFields) == 0 {
// Select all default columns
sb.SelectMore(logstelemetryschema.LogsV2TraceIDColumn)
sb.SelectMore(logstelemetryschema.LogsV2SpanIDColumn)
sb.SelectMore(logstelemetryschema.LogsV2TraceFlagsColumn)
sb.SelectMore(logstelemetryschema.LogsV2SeverityTextColumn)
sb.SelectMore(logstelemetryschema.LogsV2SeverityNumberColumn)
sb.SelectMore(logstelemetryschema.LogsV2ScopeNameColumn)
sb.SelectMore(logstelemetryschema.LogsV2ScopeVersionColumn)
sb.SelectMore(LogsV2TraceIDColumn)
sb.SelectMore(LogsV2SpanIDColumn)
sb.SelectMore(LogsV2TraceFlagsColumn)
sb.SelectMore(LogsV2SeverityTextColumn)
sb.SelectMore(LogsV2SeverityNumberColumn)
sb.SelectMore(LogsV2ScopeNameColumn)
sb.SelectMore(LogsV2ScopeVersionColumn)
sb.SelectMore(bodyAliasExpression(b.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID))))
sb.SelectMore(logstelemetryschema.LogsV2AttributesStringColumn)
sb.SelectMore(logstelemetryschema.LogsV2AttributesNumberColumn)
sb.SelectMore(logstelemetryschema.LogsV2AttributesBoolColumn)
sb.SelectMore(logstelemetryschema.LogsV2ResourcesStringColumn)
sb.SelectMore(logstelemetryschema.LogsV2ScopeStringColumn)
sb.SelectMore(LogsV2AttributesStringColumn)
sb.SelectMore(LogsV2AttributesNumberColumn)
sb.SelectMore(LogsV2AttributesBoolColumn)
sb.SelectMore(LogsV2ResourcesStringColumn)
sb.SelectMore(LogsV2ScopeStringColumn)
} else {
// Select specified columns
for index := range query.SelectFields {
if query.SelectFields[index].Name == logstelemetryschema.LogsV2TimestampColumn || query.SelectFields[index].Name == logstelemetryschema.LogsV2IDColumn {
if query.SelectFields[index].Name == LogsV2TimestampColumn || query.SelectFields[index].Name == LogsV2IDColumn {
continue
}
@@ -329,7 +318,7 @@ func (b *logQueryStatementBuilder) buildListQuery(
}
}
sb.From(fmt.Sprintf("%s.%s", logstelemetryschema.DBName, logstelemetryschema.LogsV2TableName))
sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
// Add filter conditions
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables, skipResourceFilter)
@@ -435,7 +424,7 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
}
// Add FROM clause
sb.From(fmt.Sprintf("%s.%s", logstelemetryschema.DBName, logstelemetryschema.LogsV2TableName))
sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables, skipResourceFilter)
@@ -603,7 +592,7 @@ func (b *logQueryStatementBuilder) buildScalarQuery(
}
}
sb.From(fmt.Sprintf("%s.%s", logstelemetryschema.DBName, logstelemetryschema.LogsV2TableName))
sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
// Add filter conditions
preparedWhereClause, err := b.addFilterCondition(ctx, orgID, sb, start, end, query, keys, variables, skipResourceFilter)

View File

@@ -1,4 +1,4 @@
package logstelemetryschema
package telemetrylogs
import (
"testing"

View File

@@ -1,4 +1,4 @@
package logsstatementbuilder
package telemetrylogs
import (
"context"
@@ -11,7 +11,6 @@ import (
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
@@ -213,12 +212,12 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
fl := flaggertest.New(t)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
keysMap := logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
keysMap := buildCompleteFieldKeyMap(releaseTime)
mockMetadataStore.KeysMap = keysMap
fm := logstelemetryschema.NewFieldMapper(fl)
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
fm := NewFieldMapper(fl)
cb := NewConditionBuilder(fm, fl)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
@@ -228,7 +227,7 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
fm,
cb,
aggExprRewriter,
logstelemetryschema.DefaultFullTextColumn,
DefaultFullTextColumn,
fl,
nil,
false,
@@ -356,12 +355,12 @@ func TestStatementBuilderListQuery(t *testing.T) {
ctx := context.Background()
fl := flaggertest.New(t)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
fm := logstelemetryschema.NewFieldMapper(fl)
fm := NewFieldMapper(fl)
// Create a test release time
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
cb := NewConditionBuilder(fm, fl)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
@@ -371,7 +370,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
fm,
cb,
aggExprRewriter,
logstelemetryschema.DefaultFullTextColumn,
DefaultFullTextColumn,
fl,
nil,
false,
@@ -508,11 +507,11 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
ctx := context.Background()
fl := flaggertest.New(t)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
fm := logstelemetryschema.NewFieldMapper(fl)
fm := NewFieldMapper(fl)
// Create a test release time
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
cb := NewConditionBuilder(fm, fl)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
@@ -522,7 +521,7 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
fm,
cb,
aggExprRewriter,
logstelemetryschema.DefaultFullTextColumn,
DefaultFullTextColumn,
fl,
nil,
false,
@@ -586,11 +585,11 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) {
ctx := context.Background()
fl := flaggertest.New(t)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
fm := logstelemetryschema.NewFieldMapper(fl)
fm := NewFieldMapper(fl)
// Create a test release time
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
cb := NewConditionBuilder(fm, fl)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
@@ -600,7 +599,7 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) {
fm,
cb,
aggExprRewriter,
logstelemetryschema.DefaultFullTextColumn,
DefaultFullTextColumn,
fl,
nil,
false,
@@ -685,9 +684,9 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
ctx := context.Background()
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
fl := flaggertest.New(t)
fm := logstelemetryschema.NewFieldMapper(fl)
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMapCollision()
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
fm := NewFieldMapper(fl)
mockMetadataStore.KeysMap = buildCompleteFieldKeyMapCollision()
cb := NewConditionBuilder(fm, fl)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
@@ -697,7 +696,7 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
fm,
cb,
aggExprRewriter,
logstelemetryschema.DefaultFullTextColumn,
DefaultFullTextColumn,
fl,
nil,
false,
@@ -741,8 +740,8 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
expectedKey: logstelemetryschema.IntrinsicFields["severity_text"],
keysMap: buildCompleteFieldKeyMap(releaseTime),
expectedKey: IntrinsicFields["severity_text"],
},
{
name: "intrinsic field with other key match - no override",
@@ -778,7 +777,7 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextBody,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(releaseTime),
expectedKey: telemetrytypes.TelemetryFieldKey{
Name: "severity_number",
FieldContext: telemetrytypes.FieldContextBody,
@@ -792,8 +791,8 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
expectedKey: *logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["service.name"][0],
keysMap: buildCompleteFieldKeyMap(releaseTime),
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["service.name"][0],
},
{
name: "single matching key with incorrect context specified - no override",
@@ -802,7 +801,7 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(releaseTime),
expectedKey: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
FieldContext: telemetrytypes.FieldContextAttribute,
@@ -816,8 +815,8 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
expectedKey: *logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["service.name"][0],
keysMap: buildCompleteFieldKeyMap(releaseTime),
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["service.name"][0],
},
{
name: "multiple matching keys - all materialized",
@@ -826,7 +825,7 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(releaseTime),
expectedKey: telemetrytypes.TelemetryFieldKey{
Name: "multi.mat.key",
FieldDataType: telemetrytypes.FieldDataTypeString,
@@ -840,7 +839,7 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(releaseTime),
expectedKey: telemetrytypes.TelemetryFieldKey{
Name: "mixed.materialization.key",
FieldDataType: telemetrytypes.FieldDataTypeString,
@@ -854,8 +853,8 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
expectedKey: *logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["mixed.materialization.key"][0],
keysMap: buildCompleteFieldKeyMap(releaseTime),
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["mixed.materialization.key"][0],
},
{
name: "no matching keys - unknown field",
@@ -864,7 +863,7 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(releaseTime),
expectedKey: telemetrytypes.TelemetryFieldKey{
Name: "unknown.field",
FieldContext: telemetrytypes.FieldContextUnspecified,
@@ -879,7 +878,7 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(releaseTime),
expectedKey: telemetrytypes.TelemetryFieldKey{
Name: "unknown.field",
FieldContext: telemetrytypes.FieldContextAttribute,
@@ -894,8 +893,8 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
expectedKey: *logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["mat.key"][0],
keysMap: buildCompleteFieldKeyMap(releaseTime),
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["mat.key"][0],
},
{
name: "non-materialized field",
@@ -904,16 +903,16 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime),
expectedKey: *logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)["user.id"][0],
keysMap: buildCompleteFieldKeyMap(releaseTime),
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["user.id"][0],
},
}
fl := flaggertest.New(t)
fm := logstelemetryschema.NewFieldMapper(fl)
fm := NewFieldMapper(fl)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMapCollision()
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
mockMetadataStore.KeysMap = buildCompleteFieldKeyMapCollision()
cb := NewConditionBuilder(fm, fl)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
@@ -923,7 +922,7 @@ func TestAdjustKey(t *testing.T) {
fm,
cb,
aggExprRewriter,
logstelemetryschema.DefaultFullTextColumn,
DefaultFullTextColumn,
fl,
nil,
false,
@@ -1055,11 +1054,11 @@ func TestStmtBuilderBodyField(t *testing.T) {
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
fl := flaggertest.WithUseJSONBody(t, c.enableUseJSONBody)
fm := logstelemetryschema.NewFieldMapper(fl)
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
fm := NewFieldMapper(fl)
cb := NewConditionBuilder(fm, fl)
// build the key map
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
for _, field := range logstelemetryschema.IntrinsicFields {
for _, field := range IntrinsicFields {
f := field
mockMetadataStore.KeysMap[field.Name] = append(mockMetadataStore.KeysMap[field.Name], &f)
}
@@ -1070,7 +1069,7 @@ func TestStmtBuilderBodyField(t *testing.T) {
fm,
cb,
aggExprRewriter,
logstelemetryschema.DefaultFullTextColumn,
DefaultFullTextColumn,
fl,
nil,
false,
@@ -1156,11 +1155,11 @@ func TestStmtBuilderBodyFullTextSearch(t *testing.T) {
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
fl := flaggertest.WithUseJSONBody(t, c.enableUseJSONBody)
fm := logstelemetryschema.NewFieldMapper(fl)
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
fm := NewFieldMapper(fl)
cb := NewConditionBuilder(fm, fl)
// build the key map
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
for _, field := range logstelemetryschema.IntrinsicFields {
for _, field := range IntrinsicFields {
f := field
mockMetadataStore.KeysMap[field.Name] = append(mockMetadataStore.KeysMap[field.Name], &f)
}
@@ -1171,7 +1170,7 @@ func TestStmtBuilderBodyFullTextSearch(t *testing.T) {
fm,
cb,
aggExprRewriter,
logstelemetryschema.DefaultFullTextColumn,
DefaultFullTextColumn,
fl,
nil,
false,
@@ -1274,14 +1273,14 @@ func newSkipResourceFingerprintLogsBuilder(
t.Helper()
fl := flaggertest.New(t)
fm := logstelemetryschema.NewFieldMapper(fl)
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
fm := NewFieldMapper(fl)
cb := NewConditionBuilder(fm, fl)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMap(time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC))
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC))
aggExprRewriter := querybuilder.NewAggExprRewriter(
instrumentationtest.New().ToProviderSettings(),
logstelemetryschema.DefaultFullTextColumn,
DefaultFullTextColumn,
fm,
cb,
fl,
@@ -1293,7 +1292,7 @@ func newSkipResourceFingerprintLogsBuilder(
fm,
cb,
aggExprRewriter,
logstelemetryschema.DefaultFullTextColumn,
DefaultFullTextColumn,
fl,
telemetryStore,
skipEnable,

View File

@@ -0,0 +1,15 @@
package telemetrylogs
const (
DBName = "signoz_logs"
LogsV2TableName = "distributed_logs_v2"
LogsV2LocalTableName = "logs_v2"
TagAttributesV2TableName = "distributed_tag_attributes_v2"
TagAttributesV2LocalTableName = "tag_attributes_v2"
LogAttributeKeysTblName = "distributed_logs_attribute_keys"
LogResourceKeysTblName = "distributed_logs_resource_keys"
LogsResourceV2TableName = "distributed_logs_v2_resource"
PathTypesTableName = "distributed_json_path_types"
PromotedPathsTableName = "distributed_json_promoted_paths"
SkipIndexTableName = "system.data_skipping_indices"
)

View File

@@ -1,4 +1,4 @@
package logstelemetryschema
package telemetrylogs
import (
"strings"
@@ -19,7 +19,7 @@ func limitString(s string, maxLen int) string {
}
// Function to build a complete field key map for testing all scenarios.
func BuildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytypes.TelemetryFieldKey {
func buildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytypes.TelemetryFieldKey {
keysMap := map[string][]*telemetrytypes.TelemetryFieldKey{
"service.name": {
{
@@ -953,7 +953,7 @@ func BuildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytype
return keysMap
}
func BuildCompleteFieldKeyMapCollision() map[string][]*telemetrytypes.TelemetryFieldKey {
func buildCompleteFieldKeyMapCollision() map[string][]*telemetrytypes.TelemetryFieldKey {
keysMap := map[string][]*telemetrytypes.TelemetryFieldKey{
"service.name": {
{

View File

@@ -13,7 +13,7 @@ import (
"github.com/SigNoz/signoz-otel-collector/constants"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -139,8 +139,8 @@ func buildListLogsJSONIndexesQuery(cluster string, filters ...string) (string, [
"name", "type_full", "expr", "granularity",
).From(fmt.Sprintf("clusterAllReplicas('%s', %s)", cluster, SkipIndexTableName))
sb.Where(sb.Equal("database", logstelemetryschema.DBName))
sb.Where(sb.Equal("table", logstelemetryschema.LogsV2LocalTableName))
sb.Where(sb.Equal("database", telemetrylogs.DBName))
sb.Where(sb.Equal("table", telemetrylogs.LogsV2LocalTableName))
sb.Where(sb.Or(
sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyV2ColumnPrefix))),
sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyPromotedColumnPrefix))),
@@ -189,12 +189,12 @@ func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ..
baseColumn := ""
fieldName := ""
switch {
case strings.HasPrefix(columnExpr, logstelemetryschema.BodyV2ColumnPrefix):
baseColumn = logstelemetryschema.BodyV2ColumnPrefix
fieldName = strings.TrimPrefix(columnExpr, logstelemetryschema.BodyV2ColumnPrefix)
case strings.HasPrefix(columnExpr, logstelemetryschema.BodyPromotedColumnPrefix):
baseColumn = logstelemetryschema.BodyPromotedColumnPrefix
fieldName = strings.TrimPrefix(columnExpr, logstelemetryschema.BodyPromotedColumnPrefix)
case strings.HasPrefix(columnExpr, telemetrylogs.BodyV2ColumnPrefix):
baseColumn = telemetrylogs.BodyV2ColumnPrefix
fieldName = strings.TrimPrefix(columnExpr, telemetrylogs.BodyV2ColumnPrefix)
case strings.HasPrefix(columnExpr, telemetrylogs.BodyPromotedColumnPrefix):
baseColumn = telemetrylogs.BodyPromotedColumnPrefix
fieldName = strings.TrimPrefix(columnExpr, telemetrylogs.BodyPromotedColumnPrefix)
}
fieldName = strings.ReplaceAll(fieldName, "`", "")
@@ -228,12 +228,12 @@ func (t *telemetryMetaStore) ListJSONValues(ctx context.Context, path string, li
}
if promoted {
path = logstelemetryschema.BodyPromotedColumnPrefix + path
path = telemetrylogs.BodyPromotedColumnPrefix + path
} else {
path = logstelemetryschema.BodyV2ColumnPrefix + path
path = telemetrylogs.BodyV2ColumnPrefix + path
}
from := fmt.Sprintf("%s.%s", logstelemetryschema.DBName, logstelemetryschema.LogsV2TableName)
from := fmt.Sprintf("%s.%s", telemetrylogs.DBName, telemetrylogs.LogsV2TableName)
colExpr := func(typ telemetrytypes.JSONDataType) string {
return fmt.Sprintf("dynamicElement(%s, '%s')", path, typ.StringValue())
}
@@ -382,7 +382,7 @@ func (t *telemetryMetaStore) IsPathPromoted(ctx context.Context, path string) (b
split := strings.Split(path, telemetrytypes.ArraySep)
pathSegment := split[0]
query := fmt.Sprintf("SELECT 1 FROM %s.%s WHERE signal = ? AND column_name = ? AND field_context = ? AND field_name = ? LIMIT 1", DBName, PromotedPathsTableName)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, telemetrytypes.SignalLogs, logstelemetryschema.LogsV2BodyPromotedColumn, telemetrytypes.FieldContextBody, pathSegment)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, telemetrytypes.SignalLogs, telemetrylogs.LogsV2BodyPromotedColumn, telemetrytypes.FieldContextBody, pathSegment)
if err != nil {
return false, errors.WrapInternalf(err, CodeFailCheckPathPromoted, "failed to check if path %s is promoted", path)
}
@@ -397,7 +397,7 @@ func (t *telemetryMetaStore) GetPromotedPaths(ctx context.Context, paths ...stri
sb := sqlbuilder.Select("field_name").From(fmt.Sprintf("%s.%s", DBName, PromotedPathsTableName))
conditions := []string{
sb.Equal("signal", telemetrytypes.SignalLogs),
sb.Equal("column_name", logstelemetryschema.LogsV2BodyPromotedColumn),
sb.Equal("column_name", telemetrylogs.LogsV2BodyPromotedColumn),
sb.Equal("field_context", telemetrytypes.FieldContextBody),
sb.NotEqual("field_name", "__all__"),
}
@@ -433,8 +433,8 @@ func (t *telemetryMetaStore) GetPromotedPaths(ctx context.Context, paths ...stri
// TODO(Piyush): Remove this function.
func CleanPathPrefixes(path string) string {
path = strings.TrimPrefix(path, telemetrytypes.BodyJSONStringSearchPrefix)
path = strings.TrimPrefix(path, logstelemetryschema.BodyV2ColumnPrefix)
path = strings.TrimPrefix(path, logstelemetryschema.BodyPromotedColumnPrefix)
path = strings.TrimPrefix(path, telemetrylogs.BodyV2ColumnPrefix)
path = strings.TrimPrefix(path, telemetrylogs.BodyPromotedColumnPrefix)
return path
}
@@ -454,7 +454,7 @@ func (t *telemetryMetaStore) PromotePaths(ctx context.Context, paths ...string)
if trimmed == "" {
continue
}
if err := batch.Append(telemetrytypes.SignalLogs, logstelemetryschema.LogsV2BodyPromotedColumn, "JSON()", telemetrytypes.FieldContextBody, trimmed, 0, releaseTime); err != nil {
if err := batch.Append(telemetrytypes.SignalLogs, telemetrylogs.LogsV2BodyPromotedColumn, "JSON()", telemetrytypes.FieldContextBody, trimmed, 0, releaseTime); err != nil {
_ = batch.Abort()
return errors.WrapInternalf(err, CodeFailedToAppendPath, "failed to append path")
}

View File

@@ -6,7 +6,7 @@ import (
"github.com/SigNoz/signoz-otel-collector/constants"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/stretchr/testify/require"
)
@@ -25,8 +25,8 @@ func TestBuildListLogsJSONIndexesQuery(t *testing.T) {
expectedSQL: "SELECT name, type_full, expr, granularity FROM clusterAllReplicas('test-cluster', system.data_skipping_indices) " +
"WHERE database = ? AND table = ? AND (LOWER(expr) LIKE LOWER(?) OR LOWER(expr) LIKE LOWER(?))",
expectedArgs: []any{
logstelemetryschema.DBName,
logstelemetryschema.LogsV2LocalTableName,
telemetrylogs.DBName,
telemetrylogs.LogsV2LocalTableName,
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyV2ColumnPrefix)),
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyPromotedColumnPrefix)),
},
@@ -38,8 +38,8 @@ func TestBuildListLogsJSONIndexesQuery(t *testing.T) {
expectedSQL: "SELECT name, type_full, expr, granularity FROM clusterAllReplicas('test-cluster', system.data_skipping_indices) " +
"WHERE database = ? AND table = ? AND (LOWER(expr) LIKE LOWER(?) OR LOWER(expr) LIKE LOWER(?)) AND (LOWER(replaceAll(expr, '`', '')) LIKE LOWER(?) OR LOWER(replaceAll(expr, '`', '')) LIKE LOWER(?))",
expectedArgs: []any{
logstelemetryschema.DBName,
logstelemetryschema.LogsV2LocalTableName,
telemetrylogs.DBName,
telemetrylogs.LogsV2LocalTableName,
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyV2ColumnPrefix)),
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyPromotedColumnPrefix)),
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains("foo")),

View File

@@ -14,12 +14,11 @@ import (
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryaudit"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrytraces"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/featuretypes"
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
@@ -78,6 +77,27 @@ func escapeForLike(s string) string {
func NewTelemetryMetaStore(
settings factory.ProviderSettings,
telemetrystore telemetrystore.TelemetryStore,
tracesDBName string,
tracesFieldsTblName string,
spanAttributesKeysTblName string,
indexV3TblName string,
metricsDBName string,
metricsFieldsTblName string,
meterDBName string,
meterFieldsTblName string,
logsDBName string,
logsV2TblName string,
logsFieldsTblName string,
logAttributeKeysTblName string,
logResourceKeysTblName string,
auditDBName string,
auditLogsTblName string,
auditFieldsTblName string,
auditAttributeKeysTblName string,
auditResourceKeysTblName string,
relatedMetadataDBName string,
relatedMetadataTblName string,
columnEvolutionMetadataTblName string,
fl flagger.Flagger,
) telemetrytypes.MetadataStore {
metadataSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrymetadata")
@@ -88,32 +108,32 @@ func NewTelemetryMetaStore(
t := &telemetryMetaStore{
logger: metadataSettings.Logger(),
telemetrystore: telemetrystore,
tracesDBName: tracestelemetryschema.DBName,
tracesFieldsTblName: tracestelemetryschema.TagAttributesV2TableName,
spanAttributesKeysTblName: tracestelemetryschema.SpanAttributesKeysTblName,
indexV3TblName: tracestelemetryschema.SpanIndexV3TableName,
metricsDBName: metricstelemetryschema.DBName,
metricsFieldsTblName: metricstelemetryschema.AttributesMetadataTableName,
meterDBName: metertelemetryschema.DBName,
meterFieldsTblName: metertelemetryschema.SamplesAgg1dTableName,
logsDBName: logstelemetryschema.DBName,
logsV2TblName: logstelemetryschema.LogsV2TableName,
logsFieldsTblName: logstelemetryschema.TagAttributesV2TableName,
logAttributeKeysTblName: logstelemetryschema.LogAttributeKeysTblName,
logResourceKeysTblName: logstelemetryschema.LogResourceKeysTblName,
auditDBName: audittelemetryschema.DBName,
auditLogsTblName: audittelemetryschema.AuditLogsTableName,
auditFieldsTblName: audittelemetryschema.TagAttributesTableName,
auditAttributeKeysTblName: audittelemetryschema.LogAttributeKeysTblName,
auditResourceKeysTblName: audittelemetryschema.LogResourceKeysTblName,
relatedMetadataDBName: DBName,
relatedMetadataTblName: AttributesMetadataTableName,
columnEvolutionMetadataTblName: ColumnEvolutionMetadataTableName,
tracesDBName: tracesDBName,
tracesFieldsTblName: tracesFieldsTblName,
spanAttributesKeysTblName: spanAttributesKeysTblName,
indexV3TblName: indexV3TblName,
metricsDBName: metricsDBName,
metricsFieldsTblName: metricsFieldsTblName,
meterDBName: meterDBName,
meterFieldsTblName: meterFieldsTblName,
logsDBName: logsDBName,
logsV2TblName: logsV2TblName,
logsFieldsTblName: logsFieldsTblName,
logAttributeKeysTblName: logAttributeKeysTblName,
logResourceKeysTblName: logResourceKeysTblName,
auditDBName: auditDBName,
auditLogsTblName: auditLogsTblName,
auditFieldsTblName: auditFieldsTblName,
auditAttributeKeysTblName: auditAttributeKeysTblName,
auditResourceKeysTblName: auditResourceKeysTblName,
relatedMetadataDBName: relatedMetadataDBName,
relatedMetadataTblName: relatedMetadataTblName,
columnEvolutionMetadataTblName: columnEvolutionMetadataTblName,
jsonColumnMetadata: map[telemetrytypes.Signal]map[telemetrytypes.FieldContext]telemetrytypes.JSONColumnMetadata{
telemetrytypes.SignalLogs: {
telemetrytypes.FieldContextBody: telemetrytypes.JSONColumnMetadata{
BaseColumn: logstelemetryschema.LogsV2BodyV2Column,
PromotedColumn: logstelemetryschema.LogsV2BodyPromotedColumn,
BaseColumn: telemetrylogs.LogsV2BodyV2Column,
PromotedColumn: telemetrylogs.LogsV2BodyPromotedColumn,
},
},
},
@@ -289,8 +309,8 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
complete := rowCount <= limit
staticKeys := []string{"isRoot", "isEntryPoint"}
staticKeys = append(staticKeys, maps.Keys(tracestelemetryschema.IntrinsicFields)...)
staticKeys = append(staticKeys, maps.Keys(tracestelemetryschema.CalculatedFields)...)
staticKeys = append(staticKeys, maps.Keys(telemetrytraces.IntrinsicFields)...)
staticKeys = append(staticKeys, maps.Keys(telemetrytraces.CalculatedFields)...)
// Add matching intrinsic and matching calculated fields
// These don't count towards the limit
@@ -304,14 +324,14 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
}
if found {
if field, exists := tracestelemetryschema.IntrinsicFields[key]; exists {
if field, exists := telemetrytraces.IntrinsicFields[key]; exists {
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
keys = append(keys, &field)
}
continue
}
if field, exists := tracestelemetryschema.CalculatedFields[key]; exists {
if field, exists := telemetrytraces.CalculatedFields[key]; exists {
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
keys = append(keys, &field)
}
@@ -636,7 +656,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
complete := rowCount <= limit
staticKeys := []string{}
staticKeys = append(staticKeys, maps.Keys(logstelemetryschema.IntrinsicFields)...)
staticKeys = append(staticKeys, maps.Keys(telemetrylogs.IntrinsicFields)...)
// Add matching intrinsic and matching calculated fields
// These don't count towards the limit
@@ -650,7 +670,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
}
if found {
if field, exists := logstelemetryschema.IntrinsicFields[key]; exists {
if field, exists := telemetrylogs.IntrinsicFields[key]; exists {
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
keys = append(keys, &field)
}
@@ -878,7 +898,7 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
complete := rowCount <= limit
// Add intrinsic audit fields (same as logs intrinsics: body, severity_text, etc.)
staticKeys := maps.Keys(audittelemetryschema.IntrinsicFields)
staticKeys := maps.Keys(telemetryaudit.IntrinsicFields)
for _, key := range staticKeys {
found := false
for _, v := range searchTexts {
@@ -889,7 +909,7 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
}
if found {
if field, exists := audittelemetryschema.IntrinsicFields[key]; exists {
if field, exists := telemetryaudit.IntrinsicFields[key]; exists {
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
keys = append(keys, &field)
}
@@ -1156,7 +1176,7 @@ func enrichWithIntrinsicMetricKeys(keys map[string][]*telemetrytypes.TelemetryFi
continue
}
for name, key := range metricstelemetryschema.IntrinsicMetricFieldDefinitions {
for name, key := range telemetrymetrics.IntrinsicMetricFieldDefinitions {
if !selectorMatchesIntrinsicField(selector, key) {
continue
}
@@ -1872,7 +1892,7 @@ func (t *telemetryMetaStore) getIntrinsicMetricFieldValues(ctx context.Context,
instrumentationtypes.CodeNamespace: "metadata",
instrumentationtypes.CodeFunctionName: "getIntrinsicMetricFieldValues",
})
key, ok := metricstelemetryschema.IntrinsicMetricFieldDefinitions[fieldValueSelector.Name]
key, ok := telemetrymetrics.IntrinsicMetricFieldDefinitions[fieldValueSelector.Name]
if !ok {
return &telemetrytypes.TelemetryFieldValues{}, nil
}
@@ -1902,18 +1922,18 @@ func (t *telemetryMetaStore) getIntrinsicMetricFieldValues(ctx context.Context,
tableName string
extraConds []string
}{
{metricstelemetryschema.TimeseriesV41weekTableName, nil},
{telemetrymetrics.TimeseriesV41weekTableName, nil},
}
if reductionEnabled {
sources = append(sources,
struct {
tableName string
extraConds []string
}{metricstelemetryschema.TimeseriesV4BufferTableName, []string{"is_reduced = false"}},
}{telemetrymetrics.TimeseriesV4BufferTableName, []string{"is_reduced = false"}},
struct {
tableName string
extraConds []string
}{metricstelemetryschema.TimeseriesV4ReducedTableName, nil},
}{telemetrymetrics.TimeseriesV4ReducedTableName, nil},
)
}
@@ -2242,7 +2262,7 @@ func (t *telemetryMetaStore) fetchMetricsTemporalityAndType(ctx context.Context,
instrumentationtypes.CodeNamespace: "metadata",
instrumentationtypes.CodeFunctionName: "fetchMetricsTemporalityAndType",
})
adjustedStartTs, adjustedEndTs, tsTableName, _ := metricstelemetryschema.WhichTSTableToUse(queryTimeRangeStartTs, queryTimeRangeEndTs, false, nil)
adjustedStartTs, adjustedEndTs, tsTableName, _ := telemetrymetrics.WhichTSTableToUse(queryTimeRangeStartTs, queryTimeRangeEndTs, false, nil)
temporalities, types, err := t.fetchTemporalityTypeForTable(ctx, tsTableName, adjustedStartTs, adjustedEndTs, metricNames)
if err != nil {
@@ -2251,7 +2271,7 @@ func (t *telemetryMetaStore) fetchMetricsTemporalityAndType(ctx context.Context,
reduced := make(map[string]bool)
if reductionEnabled {
bufferTemporalities, bufferTypes, err := t.fetchTemporalityTypeForTable(ctx, metricstelemetryschema.TimeseriesV4BufferTableName, adjustedStartTs, adjustedEndTs, metricNames, "is_reduced = false")
bufferTemporalities, bufferTypes, err := t.fetchTemporalityTypeForTable(ctx, telemetrymetrics.TimeseriesV4BufferTableName, adjustedStartTs, adjustedEndTs, metricNames, "is_reduced = false")
if err != nil {
return nil, nil, nil, err
}
@@ -2264,7 +2284,7 @@ func (t *telemetryMetaStore) fetchMetricsTemporalityAndType(ctx context.Context,
}
}
reducedTemporalities, reducedTypes, err := t.fetchTemporalityTypeForTable(ctx, metricstelemetryschema.TimeseriesV4ReducedTableName, adjustedStartTs, adjustedEndTs, metricNames)
reducedTemporalities, reducedTypes, err := t.fetchTemporalityTypeForTable(ctx, telemetrymetrics.TimeseriesV4ReducedTableName, adjustedStartTs, adjustedEndTs, metricNames)
if err != nil {
return nil, nil, nil, err
}
@@ -2596,13 +2616,13 @@ func (t *telemetryMetaStore) GetFirstSeenFromMetricMetadata(ctx context.Context,
}
func (t *telemetryMetaStore) FetchLastSeenInfoMulti(ctx context.Context, orgID valuer.UUID, metricNames ...string) (map[string]int64, error) {
lastSeenInfo, err := t.fetchLastSeenInfoForTable(ctx, metricstelemetryschema.TimeseriesV4TableName, metricNames)
lastSeenInfo, err := t.fetchLastSeenInfoForTable(ctx, telemetrymetrics.TimeseriesV4TableName, metricNames)
if err != nil {
return nil, err
}
if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) {
reducedLastSeen, err := t.fetchLastSeenInfoForTable(ctx, metricstelemetryschema.TimeseriesV4ReducedTableName, metricNames)
reducedLastSeen, err := t.fetchLastSeenInfoForTable(ctx, telemetrymetrics.TimeseriesV4ReducedTableName, metricNames)
if err != nil {
return nil, err
}

View File

@@ -9,8 +9,13 @@ import (
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/telemetryaudit"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/telemetrymeter"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
"github.com/SigNoz/signoz/pkg/telemetrytraces"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -37,6 +42,27 @@ func TestGetFirstSeenFromMetricMetadata(t *testing.T) {
metadata := NewTelemetryMetaStore(
instrumentationtest.New().ToProviderSettings(),
mockTelemetryStore,
telemetrytraces.DBName,
telemetrytraces.TagAttributesV2TableName,
telemetrytraces.SpanAttributesKeysTblName,
telemetrytraces.SpanIndexV3TableName,
telemetrymetrics.DBName,
telemetrymetrics.AttributesMetadataTableName,
telemetrymeter.DBName,
telemetrymeter.SamplesAgg1dTableName,
telemetrylogs.DBName,
telemetrylogs.LogsV2TableName,
telemetrylogs.TagAttributesV2TableName,
telemetrylogs.LogAttributeKeysTblName,
telemetrylogs.LogResourceKeysTblName,
telemetryaudit.DBName,
telemetryaudit.AuditLogsTableName,
telemetryaudit.TagAttributesTableName,
telemetryaudit.LogAttributeKeysTblName,
telemetryaudit.LogResourceKeysTblName,
DBName,
AttributesMetadataTableName,
ColumnEvolutionMetadataTableName,
flaggertest.New(t),
)

View File

@@ -1,4 +1,4 @@
package meterstatementbuilder
package telemetrymeter
import (
"context"
@@ -8,8 +8,7 @@ import (
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -22,7 +21,7 @@ type meterQueryStatementBuilder struct {
metadataStore telemetrytypes.MetadataStore
fm qbtypes.FieldMapper
cb qbtypes.ConditionBuilder
metricsStatementBuilder *metricsstatementbuilder.StatementBuilder
metricsStatementBuilder *telemetrymetrics.MetricQueryStatementBuilder
}
var _ qbtypes.StatementBuilder[qbtypes.MetricAggregation] = (*meterQueryStatementBuilder)(nil)
@@ -32,9 +31,9 @@ func NewMeterQueryStatementBuilder(
metadataStore telemetrytypes.MetadataStore,
fieldMapper qbtypes.FieldMapper,
conditionBuilder qbtypes.ConditionBuilder,
metricsStatementBuilder *metricsstatementbuilder.StatementBuilder,
metricsStatementBuilder *telemetrymetrics.MetricQueryStatementBuilder,
) *meterQueryStatementBuilder {
metricsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema")
metricsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrymeter")
return &meterQueryStatementBuilder{
logger: metricsSettings.Logger(),
@@ -54,7 +53,7 @@ func (b *meterQueryStatementBuilder) Build(
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
keySelectors := metricsstatementbuilder.GetKeySelectors(query)
keySelectors := telemetrymetrics.GetKeySelectors(query)
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, keySelectors)
if err != nil {
return nil, err
@@ -134,8 +133,8 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath(
sb.SelectMore(col)
}
tbl := metertelemetryschema.WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
aggCol, err := metertelemetryschema.AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
tbl := WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
aggCol, err := AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
if err != nil {
return "", nil, err
}
@@ -144,7 +143,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath(
}
sb.SelectMore(fmt.Sprintf("%s AS value", aggCol))
sb.From(fmt.Sprintf("%s.%s AS points", metertelemetryschema.DBName, tbl))
sb.From(fmt.Sprintf("%s.%s AS points", DBName, tbl))
sb.Where(
sb.In("metric_name", query.Aggregations[0].MetricName),
sb.GTE("unix_milli", start),
@@ -223,8 +222,8 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDelta(
sb.SelectMore(col)
}
tbl := metertelemetryschema.WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
aggCol, err := metertelemetryschema.AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality,
tbl := WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
aggCol, err := AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality,
query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
if err != nil {
return "", nil, err
@@ -235,7 +234,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDelta(
sb.SelectMore(fmt.Sprintf("%s AS per_series_value", aggCol))
sb.From(fmt.Sprintf("%s.%s AS points", metertelemetryschema.DBName, tbl))
sb.From(fmt.Sprintf("%s.%s AS points", DBName, tbl))
sb.Where(
sb.In("metric_name", query.Aggregations[0].MetricName),
sb.GTE("unix_milli", start),
@@ -301,14 +300,14 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
baseSb.SelectMore(col)
}
tbl := metertelemetryschema.WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
aggCol, err := metertelemetryschema.AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
tbl := WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
aggCol, err := AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
if err != nil {
return "", nil, err
}
baseSb.SelectMore(fmt.Sprintf("%s AS per_series_value", aggCol))
baseSb.From(fmt.Sprintf("%s.%s AS points", metertelemetryschema.DBName, tbl))
baseSb.From(fmt.Sprintf("%s.%s AS points", DBName, tbl))
baseSb.Where(
baseSb.In("metric_name", query.Aggregations[0].MetricName),
baseSb.GTE("unix_milli", start),
@@ -351,7 +350,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
for _, g := range query.GroupBy {
wrapped.SelectMore(fmt.Sprintf("`%s`", g.Name))
}
wrapped.SelectMore(fmt.Sprintf("%s AS per_series_value", metricsstatementbuilder.RateTmpl))
wrapped.SelectMore(fmt.Sprintf("%s AS per_series_value", telemetrymetrics.RateTmpl))
wrapped.From(fmt.Sprintf("(%s) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)", innerQuery))
q, args := wrapped.BuildWithFlavor(sqlbuilder.ClickHouse, innerArgs...)
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, nil
@@ -362,7 +361,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
for _, g := range query.GroupBy {
wrapped.SelectMore(fmt.Sprintf("`%s`", g.Name))
}
wrapped.SelectMore(fmt.Sprintf("%s AS per_series_value", metricsstatementbuilder.IncreaseTmpl))
wrapped.SelectMore(fmt.Sprintf("%s AS per_series_value", telemetrymetrics.IncreaseTmpl))
wrapped.From(fmt.Sprintf("(%s) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)", innerQuery))
q, args := wrapped.BuildWithFlavor(sqlbuilder.ClickHouse, innerArgs...)
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, nil

View File

@@ -1,4 +1,4 @@
package meterstatementbuilder
package telemetrymeter
import (
"context"
@@ -7,8 +7,7 @@ import (
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -159,8 +158,8 @@ func TestStatementBuilder(t *testing.T) {
},
}
fm := metricstelemetryschema.NewFieldMapper()
cb := metricstelemetryschema.NewConditionBuilder(fm)
fm := telemetrymetrics.NewFieldMapper()
cb := telemetrymetrics.NewConditionBuilder(fm)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
keys, err := telemetrytypestest.LoadFieldKeysFromJSON("testdata/keys_map.json")
if err != nil {
@@ -170,7 +169,7 @@ func TestStatementBuilder(t *testing.T) {
flagger := flaggertest.New(t)
metricStmtBuilder := metricsstatementbuilder.NewMetricQueryStatementBuilder(instrumentationtest.New().ToProviderSettings(), mockMetadataStore, fm, cb, flagger)
metricStmtBuilder := telemetrymetrics.NewMetricQueryStatementBuilder(instrumentationtest.New().ToProviderSettings(), mockMetadataStore, fm, cb, flagger)
statementBuilder := NewMeterQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),

View File

@@ -1,4 +1,4 @@
package metertelemetryschema
package telemetrymeter
import (
"time"

View File

@@ -1,4 +1,4 @@
package metricstelemetryschema
package telemetrymetrics
import (
"context"

View File

@@ -1,4 +1,4 @@
package metricstelemetryschema
package telemetrymetrics
import (
"context"

View File

@@ -0,0 +1,37 @@
package telemetrymetrics
import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
var IntrinsicFields = []string{
"temporality",
"metric_name",
"type",
"is_monotonic",
}
var IntrinsicMetricFieldDefinitions = map[string]telemetrytypes.TelemetryFieldKey{
"metric_name": {
Name: "metric_name",
Signal: telemetrytypes.SignalMetrics,
FieldContext: telemetrytypes.FieldContextMetric,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
// "type": {
// Name: "type",
// Signal: telemetrytypes.SignalMetrics,
// FieldContext: telemetrytypes.FieldContextMetric,
// FieldDataType: telemetrytypes.FieldDataTypeString,
// },
// "temporality": {
// Name: "temporality",
// Signal: telemetrytypes.SignalMetrics,
// FieldContext: telemetrytypes.FieldContextMetric,
// FieldDataType: telemetrytypes.FieldDataTypeString,
// },
// "is_monotonic": {
// Name: "is_monotonic",
// Signal: telemetrytypes.SignalMetrics,
// FieldContext: telemetrytypes.FieldContextMetric,
// FieldDataType: telemetrytypes.FieldDataTypeBool,
// },
}

View File

@@ -1,4 +1,4 @@
package metricstelemetryschema
package telemetrymetrics
import (
"context"

View File

@@ -1,4 +1,4 @@
package metricstelemetryschema
package telemetrymetrics
import (
"context"

View File

@@ -1,4 +1,4 @@
package metricsstatementbuilder
package telemetrymetrics
import (
"context"
@@ -7,7 +7,6 @@ import (
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -119,8 +118,8 @@ func TestReducedStatementBuilder(t *testing.T) {
},
}
fm := metricstelemetryschema.NewFieldMapper()
cb := metricstelemetryschema.NewConditionBuilder(fm)
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
fl, err := flagger.New(context.Background(), instrumentationtest.New().ToProviderSettings(), flagger.Config{}, flagger.MustNewRegistry())
require.NoError(t, err)
sb := NewMetricQueryStatementBuilder(instrumentationtest.New().ToProviderSettings(), telemetrytypestest.NewMockMetadataStore(), fm, cb, fl)

View File

@@ -1,4 +1,4 @@
package metricsstatementbuilder
package telemetrymetrics
import (
"context"
@@ -9,7 +9,6 @@ import (
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -30,7 +29,7 @@ const (
OthersMultiTemporality = `IF(LOWER(temporality) LIKE LOWER('delta'), %s, %s) AS per_series_value`
)
type StatementBuilder struct {
type MetricQueryStatementBuilder struct {
logger *slog.Logger
metadataStore telemetrytypes.MetadataStore
fm qbtypes.FieldMapper
@@ -38,7 +37,7 @@ type StatementBuilder struct {
flagger flagger.Flagger
}
var _ qbtypes.StatementBuilder[qbtypes.MetricAggregation] = (*StatementBuilder)(nil)
var _ qbtypes.StatementBuilder[qbtypes.MetricAggregation] = (*MetricQueryStatementBuilder)(nil)
func NewMetricQueryStatementBuilder(
settings factory.ProviderSettings,
@@ -46,9 +45,9 @@ func NewMetricQueryStatementBuilder(
fieldMapper qbtypes.FieldMapper,
conditionBuilder qbtypes.ConditionBuilder,
flagger flagger.Flagger,
) *StatementBuilder {
metricsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema")
return &StatementBuilder{
) *MetricQueryStatementBuilder {
metricsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrymetrics")
return &MetricQueryStatementBuilder{
logger: metricsSettings.Logger(),
metadataStore: metadataStore,
fm: fieldMapper,
@@ -90,7 +89,7 @@ func GetKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation])
return keySelectors
}
func (b *StatementBuilder) Build(
func (b *MetricQueryStatementBuilder) Build(
ctx context.Context,
orgID valuer.UUID,
start uint64,
@@ -127,7 +126,7 @@ func (b *StatementBuilder) Build(
return b.buildPipelineStatement(ctx, orgID, start, end, query, keys, variables)
}
func (b *StatementBuilder) buildPipelineStatement(
func (b *MetricQueryStatementBuilder) buildPipelineStatement(
ctx context.Context,
orgID valuer.UUID,
start, end uint64,
@@ -189,11 +188,11 @@ func (b *StatementBuilder) buildPipelineStatement(
// shaped exactly like samples_v4 / time_series_v4, so once the table names are
// chosen the rest of the pipeline is unchanged.
useBuffer := agg.Reduced &&
end-start < metricstelemetryschema.OneDayInMilliseconds &&
start >= uint64(time.Now().UnixMilli())-metricstelemetryschema.OneDayInMilliseconds
end-start < oneDayInMilliseconds &&
start >= uint64(time.Now().UnixMilli())-oneDayInMilliseconds
samplesTable, _ := metricstelemetryschema.WhichSamplesTableToUse(start, end, agg.Type, agg.TimeAggregation, useBuffer, agg.TableHints)
tsStart, tsEnd, _, tsTable := metricstelemetryschema.WhichTSTableToUse(start, end, useBuffer, agg.TableHints)
samplesTable, _ := WhichSamplesTableToUse(start, end, agg.Type, agg.TimeAggregation, useBuffer, agg.TableHints)
tsStart, tsEnd, _, tsTable := WhichTSTableToUse(start, end, useBuffer, agg.TableHints)
var timeSeriesCTE string
var timeSeriesCTEArgs []any
@@ -233,7 +232,7 @@ func (b *StatementBuilder) buildPipelineStatement(
var tsCTE string
var tsArgs []any
// time series rows are written on hour boundaries
tsStart := start - (start % metricstelemetryschema.OneHourInMilliseconds)
tsStart := start - (start % oneHourInMilliseconds)
if tsCTE, tsArgs, err = b.buildReducedTimeSeriesCTE(ctx, orgID, tsStart, end, query, keys, variables); err != nil {
return nil, err
}
@@ -283,7 +282,7 @@ func unionStatements(main, reduced *qbtypes.Statement, query qbtypes.QueryBuilde
return &qbtypes.Statement{Query: q, Args: args, Warnings: warnings}, nil
}
func (b *StatementBuilder) buildReducedTimeSeriesCTE(
func (b *MetricQueryStatementBuilder) buildReducedTimeSeriesCTE(
ctx context.Context,
orgID valuer.UUID,
start, end uint64,
@@ -313,7 +312,7 @@ func (b *StatementBuilder) buildReducedTimeSeriesCTE(
}
}
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, metricstelemetryschema.TimeseriesV4ReducedLocalTableName))
sb.From(fmt.Sprintf("%s.%s", DBName, TimeseriesV4ReducedLocalTableName))
sb.Select("fingerprint")
for _, g := range query.GroupBy {
col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys)
@@ -344,7 +343,7 @@ func (b *StatementBuilder) buildReducedTimeSeriesCTE(
// spatial_aggregation_cte in one level with no per-series grouping, so shards
// send one state per (step, group) instead of per (series, step, group).
// FINAL still dedups recomputed 60s buckets at scan time.
func (b *StatementBuilder) buildReducedSpatialAggFastPath(
func (b *MetricQueryStatementBuilder) buildReducedSpatialAggFastPath(
start, end uint64,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
timeSeriesCTE string,
@@ -353,7 +352,7 @@ func (b *StatementBuilder) buildReducedSpatialAggFastPath(
agg := query.Aggregations[0]
stepSec := int64(query.StepInterval.Seconds())
value, _, ok := metricstelemetryschema.ReducedValueColumn(agg.Type, agg.SpaceAggregation)
value, _, ok := ReducedValueColumn(agg.Type, agg.SpaceAggregation)
if !ok {
return "", nil, false
}
@@ -363,8 +362,8 @@ func (b *StatementBuilder) buildReducedSpatialAggFastPath(
for _, g := range query.GroupBy {
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
}
sb.SelectMore(fmt.Sprintf("%s AS value", metricstelemetryschema.ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec, value)))
sb.From(fmt.Sprintf("%s.%s AS points FINAL", metricstelemetryschema.DBName, metricstelemetryschema.WhichReducedSamplesTableToUse(agg.Type)))
sb.SelectMore(fmt.Sprintf("%s AS value", ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec, value)))
sb.From(fmt.Sprintf("%s.%s AS points FINAL", DBName, WhichReducedSamplesTableToUse(agg.Type)))
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.reduced_fingerprint = filtered_time_series.fingerprint")
sb.Where(
sb.In("metric_name", agg.MetricName),
@@ -378,7 +377,7 @@ func (b *StatementBuilder) buildReducedSpatialAggFastPath(
return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args, true
}
func (b *StatementBuilder) buildReducedTemporalAggregationCTE(
func (b *MetricQueryStatementBuilder) buildReducedTemporalAggregationCTE(
start, end uint64,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
timeSeriesCTE string,
@@ -387,7 +386,7 @@ func (b *StatementBuilder) buildReducedTemporalAggregationCTE(
agg := query.Aggregations[0]
stepSec := int64(query.StepInterval.Seconds())
value, weight, ok := metricstelemetryschema.ReducedValueColumn(agg.Type, agg.SpaceAggregation)
value, weight, ok := ReducedValueColumn(agg.Type, agg.SpaceAggregation)
if !ok {
return "", nil, false
}
@@ -400,13 +399,13 @@ func (b *StatementBuilder) buildReducedTemporalAggregationCTE(
for _, g := range query.GroupBy {
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
}
sb.SelectMore(fmt.Sprintf("%s AS per_series_value", metricstelemetryschema.ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec, value)))
sb.SelectMore(fmt.Sprintf("%s AS per_series_value", ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec, value)))
if weight != "" {
// count_series is a series count, not additive over time, so the avg
// denominator is reduced with avg
sb.SelectMore(fmt.Sprintf("avg(%s) AS per_series_weight", weight))
}
sb.From(fmt.Sprintf("%s.%s AS points FINAL", metricstelemetryschema.DBName, metricstelemetryschema.WhichReducedSamplesTableToUse(agg.Type)))
sb.From(fmt.Sprintf("%s.%s AS points FINAL", DBName, WhichReducedSamplesTableToUse(agg.Type)))
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.reduced_fingerprint = filtered_time_series.fingerprint")
sb.Where(
sb.In("metric_name", agg.MetricName),
@@ -420,7 +419,7 @@ func (b *StatementBuilder) buildReducedTemporalAggregationCTE(
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, true
}
func (b *StatementBuilder) buildReducedSpatialAggregationCTE(
func (b *MetricQueryStatementBuilder) buildReducedSpatialAggregationCTE(
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
) (string, []any) {
spatial := "sum(per_series_value)"
@@ -447,7 +446,7 @@ func (b *StatementBuilder) buildReducedSpatialAggregationCTE(
return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args
}
func (b *StatementBuilder) buildTemporalAggDeltaFastPath(
func (b *MetricQueryStatementBuilder) buildTemporalAggDeltaFastPath(
start, end uint64,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
samplesTable string,
@@ -466,7 +465,7 @@ func (b *StatementBuilder) buildTemporalAggDeltaFastPath(
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
}
aggCol, err := metricstelemetryschema.AggregationColumnForSamplesTable(
aggCol, err := AggregationColumnForSamplesTable(
samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation,
)
if err != nil {
@@ -484,7 +483,7 @@ func (b *StatementBuilder) buildTemporalAggDeltaFastPath(
sb.SelectMore(fmt.Sprintf("%s AS value", aggCol))
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
sb.From(fmt.Sprintf("%s.%s AS points", DBName, samplesTable))
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
sb.Where(
sb.In("metric_name", query.Aggregations[0].MetricName),
@@ -498,7 +497,7 @@ func (b *StatementBuilder) buildTemporalAggDeltaFastPath(
return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args, nil
}
func (b *StatementBuilder) buildTimeSeriesCTE(
func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE(
ctx context.Context,
orgID valuer.UUID,
start, end uint64,
@@ -530,7 +529,7 @@ func (b *StatementBuilder) buildTimeSeriesCTE(
}
}
sb.From(fmt.Sprintf("%s.%s", metricstelemetryschema.DBName, tsTable))
sb.From(fmt.Sprintf("%s.%s", DBName, tsTable))
sb.Select("fingerprint")
for _, g := range query.GroupBy {
@@ -553,7 +552,7 @@ func (b *StatementBuilder) buildTimeSeriesCTE(
// the buffer holds both raw rows and the reduced catalog rows; the raw read
// only wants the original series
if tsTable == metricstelemetryschema.TimeseriesV4BufferLocalTableName {
if tsTable == TimeseriesV4BufferLocalTableName {
sb.Where(sb.EQ("is_reduced", false))
}
@@ -568,7 +567,7 @@ func (b *StatementBuilder) buildTimeSeriesCTE(
return fmt.Sprintf("(%s) AS filtered_time_series", q), args, nil
}
func (b *StatementBuilder) buildTemporalAggregationCTE(
func (b *MetricQueryStatementBuilder) buildTemporalAggregationCTE(
ctx context.Context,
start, end uint64,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
@@ -585,7 +584,7 @@ func (b *StatementBuilder) buildTemporalAggregationCTE(
return b.buildTemporalAggForMultipleTemporalities(ctx, start, end, query, samplesTable, timeSeriesCTE, timeSeriesCTEArgs)
}
func (b *StatementBuilder) buildTemporalAggDelta(
func (b *MetricQueryStatementBuilder) buildTemporalAggDelta(
_ context.Context,
start, end uint64,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
@@ -606,7 +605,7 @@ func (b *StatementBuilder) buildTemporalAggDelta(
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
}
aggCol, err := metricstelemetryschema.AggregationColumnForSamplesTable(samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation)
aggCol, err := AggregationColumnForSamplesTable(samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation)
if err != nil {
return "", nil, err
}
@@ -617,7 +616,7 @@ func (b *StatementBuilder) buildTemporalAggDelta(
sb.SelectMore(fmt.Sprintf("%s AS per_series_value", aggCol))
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
sb.From(fmt.Sprintf("%s.%s AS points", DBName, samplesTable))
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
sb.Where(
sb.In("metric_name", query.Aggregations[0].MetricName),
@@ -632,7 +631,7 @@ func (b *StatementBuilder) buildTemporalAggDelta(
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, nil
}
func (b *StatementBuilder) buildTemporalAggCumulativeOrUnspecified(
func (b *MetricQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
_ context.Context,
start, end uint64,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
@@ -652,13 +651,13 @@ func (b *StatementBuilder) buildTemporalAggCumulativeOrUnspecified(
baseSb.SelectMore(fmt.Sprintf("`%s`", g.Name))
}
aggCol, err := metricstelemetryschema.AggregationColumnForSamplesTable(samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation)
aggCol, err := AggregationColumnForSamplesTable(samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation)
if err != nil {
return "", nil, err
}
baseSb.SelectMore(fmt.Sprintf("%s AS per_series_value", aggCol))
baseSb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
baseSb.From(fmt.Sprintf("%s.%s AS points", DBName, samplesTable))
baseSb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
baseSb.Where(
baseSb.In("metric_name", query.Aggregations[0].MetricName),
@@ -698,7 +697,7 @@ func (b *StatementBuilder) buildTemporalAggCumulativeOrUnspecified(
}
}
func (b *StatementBuilder) buildTemporalAggForMultipleTemporalities(
func (b *MetricQueryStatementBuilder) buildTemporalAggForMultipleTemporalities(
_ context.Context,
start, end uint64,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
@@ -717,11 +716,11 @@ func (b *StatementBuilder) buildTemporalAggForMultipleTemporalities(
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
}
aggForDeltaTemporality, err := metricstelemetryschema.AggregationColumnForSamplesTable(samplesTable, metrictypes.Delta, query.Aggregations[0].TimeAggregation)
aggForDeltaTemporality, err := AggregationColumnForSamplesTable(samplesTable, metrictypes.Delta, query.Aggregations[0].TimeAggregation)
if err != nil {
return "", nil, err
}
aggForCumulativeTemporality, err := metricstelemetryschema.AggregationColumnForSamplesTable(samplesTable, metrictypes.Cumulative, query.Aggregations[0].TimeAggregation)
aggForCumulativeTemporality, err := AggregationColumnForSamplesTable(samplesTable, metrictypes.Cumulative, query.Aggregations[0].TimeAggregation)
if err != nil {
return "", nil, err
}
@@ -749,7 +748,7 @@ func (b *StatementBuilder) buildTemporalAggForMultipleTemporalities(
sb.SelectMore(expr)
}
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
sb.From(fmt.Sprintf("%s.%s AS points", DBName, samplesTable))
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
sb.Where(
sb.In("metric_name", query.Aggregations[0].MetricName),
@@ -763,7 +762,7 @@ func (b *StatementBuilder) buildTemporalAggForMultipleTemporalities(
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", queryWithWindowAndOrder), args, nil
}
func (b *StatementBuilder) buildSpatialAggregationCTE(
func (b *MetricQueryStatementBuilder) buildSpatialAggregationCTE(
_ context.Context,
_ uint64,
_ uint64,
@@ -789,7 +788,7 @@ func (b *StatementBuilder) buildSpatialAggregationCTE(
return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args
}
func (b *StatementBuilder) BuildFinalSelect(
func (b *MetricQueryStatementBuilder) BuildFinalSelect(
cteFragments []string,
cteArgs [][]any,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
@@ -834,7 +833,7 @@ func (b *StatementBuilder) BuildFinalSelect(
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
}
aggQuery, err := metricstelemetryschema.AggregationQueryForHistogramCountWithParams(query.Aggregations[0].ComparisonSpaceAggregationParam)
aggQuery, err := AggregationQueryForHistogramCountWithParams(query.Aggregations[0].ComparisonSpaceAggregationParam)
if err != nil {
return nil, err
}

View File

@@ -1,4 +1,4 @@
package metricsstatementbuilder
package telemetrymetrics
import (
"context"
@@ -7,7 +7,6 @@ import (
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -253,8 +252,8 @@ func TestStatementBuilder(t *testing.T) {
},
}
fm := metricstelemetryschema.NewFieldMapper()
cb := metricstelemetryschema.NewConditionBuilder(fm)
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
keys, err := telemetrytypestest.LoadFieldKeysFromJSON("testdata/keys_map.json")
if err != nil {

View File

@@ -1,4 +1,4 @@
package metricstelemetryschema
package telemetrymetrics
import (
"fmt"
@@ -6,7 +6,6 @@ import (
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
const (
@@ -47,10 +46,10 @@ const (
)
var (
OneHourInMilliseconds = uint64(time.Hour.Milliseconds() * 1)
oneHourInMilliseconds = uint64(time.Hour.Milliseconds() * 1)
sixHoursInMilliseconds = uint64(time.Hour.Milliseconds() * 6)
OneDayInMilliseconds = uint64(time.Hour.Milliseconds() * 24)
oneWeekInMilliseconds = uint64(OneDayInMilliseconds * 7)
oneDayInMilliseconds = uint64(time.Hour.Milliseconds() * 24)
oneWeekInMilliseconds = uint64(oneDayInMilliseconds * 7)
// when the query requests for almost 1 day, but not exactly 1 day, we need to add an offset to the end time
// to make sure that we are using the correct table
@@ -69,7 +68,7 @@ func WhichTSTableToUse(
// the buffer holds the recent raw window for reduced metrics and has the same
// shape as time_series_v4; round the start to the hour like the v4 table.
if useBuffer {
start = start - (start % (OneHourInMilliseconds))
start = start - (start % (oneHourInMilliseconds))
return start, end, TimeseriesV4BufferTableName, TimeseriesV4BufferLocalTableName
}
@@ -81,7 +80,7 @@ func WhichTSTableToUse(
switch tableHints.TimeSeriesTableName {
case TimeseriesV4LocalTableName:
// adjust the start time to nearest 1 hour
start = start - (start % (OneHourInMilliseconds))
start = start - (start % (oneHourInMilliseconds))
distributedTableName = TimeseriesV4TableName
case TimeseriesV46hrsLocalTableName:
// adjust the start time to nearest 6 hours
@@ -89,7 +88,7 @@ func WhichTSTableToUse(
distributedTableName = TimeseriesV46hrsTableName
case TimeseriesV41dayLocalTableName:
// adjust the start time to nearest 1 day
start = start - (start % (OneDayInMilliseconds))
start = start - (start % (oneDayInMilliseconds))
distributedTableName = TimeseriesV41dayTableName
case TimeseriesV41weekLocalTableName:
// adjust the start time to nearest 1 week
@@ -108,17 +107,17 @@ func WhichTSTableToUse(
var localTableName string
if end-start < sixHoursInMilliseconds {
// adjust the start time to nearest 1 hour
start = start - (start % (OneHourInMilliseconds))
start = start - (start % (oneHourInMilliseconds))
distributedTableName = TimeseriesV4TableName
localTableName = TimeseriesV4LocalTableName
} else if end-start < OneDayInMilliseconds {
} else if end-start < oneDayInMilliseconds {
// adjust the start time to nearest 6 hours
start = start - (start % (sixHoursInMilliseconds))
distributedTableName = TimeseriesV46hrsTableName
localTableName = TimeseriesV46hrsLocalTableName
} else if end-start < oneWeekInMilliseconds {
// adjust the start time to nearest 1 day
start = start - (start % (OneDayInMilliseconds))
start = start - (start % (oneDayInMilliseconds))
distributedTableName = TimeseriesV41dayTableName
localTableName = TimeseriesV41dayLocalTableName
} else {
@@ -207,7 +206,7 @@ func WhichSamplesTableToUse(
return SamplesV4TableName, SamplesV4LocalTableName
}
if end-start < OneDayInMilliseconds+offsetBucket {
if end-start < oneDayInMilliseconds+offsetBucket {
return SamplesV4TableName, SamplesV4LocalTableName
} else if end-start < oneWeekInMilliseconds+offsetBucket {
return SamplesV4Agg5mTableName, SamplesV4Agg5mLocalTableName
@@ -431,37 +430,3 @@ func AggregationQueryForHistogramCountWithParams(param *metrictypes.ComparisonSp
}
}
var IntrinsicFields = []string{
"temporality",
"metric_name",
"type",
"is_monotonic",
}
var IntrinsicMetricFieldDefinitions = map[string]telemetrytypes.TelemetryFieldKey{
"metric_name": {
Name: "metric_name",
Signal: telemetrytypes.SignalMetrics,
FieldContext: telemetrytypes.FieldContextMetric,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
// "type": {
// Name: "type",
// Signal: telemetrytypes.SignalMetrics,
// FieldContext: telemetrytypes.FieldContextMetric,
// FieldDataType: telemetrytypes.FieldDataTypeString,
// },
// "temporality": {
// Name: "temporality",
// Signal: telemetrytypes.SignalMetrics,
// FieldContext: telemetrytypes.FieldContextMetric,
// FieldDataType: telemetrytypes.FieldDataTypeString,
// },
// "is_monotonic": {
// Name: "is_monotonic",
// Signal: telemetrytypes.SignalMetrics,
// FieldContext: telemetrytypes.FieldContextMetric,
// FieldDataType: telemetrytypes.FieldDataTypeBool,
// },
}

View File

@@ -1,4 +1,4 @@
package resourcefilter
package telemetryresourcefilter
import (
"context"

View File

@@ -1,4 +1,4 @@
package resourcefilter
package telemetryresourcefilter
import (
"context"

View File

@@ -1,4 +1,4 @@
package resourcefilter
package telemetryresourcefilter
import (
"context"

View File

@@ -1,4 +1,4 @@
package resourcefilter
package telemetryresourcefilter
import (
"context"

View File

@@ -1,4 +1,4 @@
package resourcefilter
package telemetryresourcefilter
import (
"context"
@@ -45,7 +45,7 @@ func New[T any](
fullTextColumn *telemetrytypes.TelemetryFieldKey,
fl flagger.Flagger,
) *resourceFilterStatementBuilder[T] {
set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/statementbuilder/resourcefilter")
set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryresourcefilter")
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
return &resourceFilterStatementBuilder[T]{

View File

@@ -1,4 +1,4 @@
package resourcefilter
package telemetryresourcefilter
import (
"context"

Some files were not shown because too many files have changed in this diff Show More