Compare commits

..

7 Commits

28 changed files with 1260 additions and 166 deletions

View File

@@ -16,6 +16,14 @@ packages:
filename: "maintenance.go"
structname: 'Mock{{.InterfaceName}}'
pkgname: '{{.SrcPackageName}}test'
github.com/SigNoz/signoz/pkg/types/condensedmetertypes:
interfaces:
Store:
config:
dir: '{{.InterfaceDir}}/condensedmetertypestest'
filename: "store.go"
structname: 'Mock{{.InterfaceName}}'
pkgname: '{{.SrcPackageName}}test'
github.com/SigNoz/signoz/pkg/types/telemetrytypes:
interfaces:
SchemaMetadataStore:

View File

@@ -16,6 +16,7 @@ import (
"github.com/SigNoz/signoz/pkg/authz/openfgaschema"
"github.com/SigNoz/signoz/pkg/authz/openfgaserver"
"github.com/SigNoz/signoz/pkg/cache"
"github.com/SigNoz/signoz/pkg/condensedmeter"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
@@ -121,6 +122,9 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
func(_ context.Context, _ factory.ProviderSettings, _ flagger.Flagger, _ licensing.Licensing, _ telemetrystore.TelemetryStore, _ retention.Getter, _ organization.Getter, _ zeus.Zeus) (factory.NamedMap[factory.ProviderFactory[meterreporter.Reporter, meterreporter.Config]], string) {
return signoz.NewMeterReporterProviderFactories(), "noop"
},
func(_ flagger.Flagger, _ licensing.Licensing, _ organization.Getter, _ telemetrystore.TelemetryStore, _ sqlstore.SQLStore) factory.NamedMap[factory.ProviderFactory[condensedmeter.Reporter, condensedmeter.Config]] {
return signoz.NewCondensedMeterProviderFactories()
},
func(ps factory.ProviderSettings, q querier.Querier, a analytics.Analytics) querier.Handler {
return querier.NewHandler(ps, q, a)
},

View File

@@ -16,6 +16,7 @@ import (
"github.com/SigNoz/signoz/ee/authz/openfgaauthz"
"github.com/SigNoz/signoz/ee/authz/openfgaschema"
"github.com/SigNoz/signoz/ee/authz/openfgaserver"
"github.com/SigNoz/signoz/ee/condensedmeter/kafkacondensedmeter"
"github.com/SigNoz/signoz/ee/gateway/httpgateway"
enterpriselicensing "github.com/SigNoz/signoz/ee/licensing"
"github.com/SigNoz/signoz/ee/licensing/httplicensing"
@@ -38,6 +39,7 @@ import (
"github.com/SigNoz/signoz/pkg/authn"
"github.com/SigNoz/signoz/pkg/authz"
"github.com/SigNoz/signoz/pkg/cache"
"github.com/SigNoz/signoz/pkg/condensedmeter"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
pkgflagger "github.com/SigNoz/signoz/pkg/flagger"
@@ -176,6 +178,13 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
return factories, "http"
},
func(flagger pkgflagger.Flagger, licensing licensing.Licensing, orgGetter organization.Getter, telemetryStore telemetrystore.TelemetryStore, sqlstore sqlstore.SQLStore) factory.NamedMap[factory.ProviderFactory[condensedmeter.Reporter, condensedmeter.Config]] {
factories := signoz.NewCondensedMeterProviderFactories()
if err := factories.Add(kafkacondensedmeter.NewFactory(flagger, licensing, orgGetter, telemetryStore, sqlstore, version.Info)); err != nil {
panic(err)
}
return factories
},
func(ps factory.ProviderSettings, q querier.Querier, a analytics.Analytics) querier.Handler {
communityHandler := querier.NewHandler(ps, q, a)
return eequerier.NewHandler(ps, q, communityHandler)

View File

@@ -522,3 +522,30 @@ meterreporter:
# interval - random(0, jitter). Must be between 10m and interval. Defaults to
# min(interval, 2h) when unset.
jitter: 2h
##################### Condensed Meter #####################
condensedmeter:
# Specifies the condensed meter provider to use.
# noop: reports nothing (community default).
# kafka: sends the hourly count of condensed metric rows to a Kafka topic as OTLP protobuf (enterprise).
provider: noop
# How often closed hours are looked for. Minimum 10m, maximum 1h.
interval: 1h
# The meter connector's dimensions. Condensed rows are split by them like every other meter row.
dimensions:
- service.name
- deployment.environment
- host.name
- signoz.workspace.key.id
# The maximum number of data points per Kafka record.
batch_size: 100
kafka:
# The seed broker addresses (host:port) used to discover the cluster.
brokers:
- localhost:9092
# The topic condensed rows are written to. It must already exist.
topic: ""
# The client id the producer reports to the brokers.
client_id: signoz
# The compression codec for produced batches: none, gzip, snappy, lz4 or zstd.
compression: lz4

View File

@@ -0,0 +1,97 @@
package kafkacondensedmeter
import (
"context"
"fmt"
"strings"
"time"
"github.com/SigNoz/signoz/pkg/clickhousesql"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/condensedmetertypes"
)
const metricNameMetricsDataPointsIngestedCount = "signoz.meter.metric.datapoint.ingested.count"
type clickhouseStore struct {
telemetryStore telemetrystore.TelemetryStore
}
func newClickhouseStore(telemetryStore telemetrystore.TelemetryStore) *clickhouseStore {
return &clickhouseStore{telemetryStore: telemetryStore}
}
// FirstIngested is the time of the first ingested meter row, zero when there is none yet.
func (store *clickhouseStore) FirstIngested(ctx context.Context) (time.Time, error) {
query := fmt.Sprintf("SELECT toInt64(ifNull(min(unix_milli), 0)) FROM %s.%s WHERE metric_name = ?", metertelemetryschema.DBName, metertelemetryschema.SamplesTableName)
var unixMilli int64
if err := store.telemetryStore.ClickhouseDB().QueryRow(ctx, query, metricNameMetricsDataPointsIngestedCount).Scan(&unixMilli); err != nil {
return time.Time{}, errors.WrapInternalf(err, errors.CodeInternal, "failed to query first ingested meter row")
}
if unixMilli == 0 {
return time.Time{}, nil
}
return time.UnixMilli(unixMilli), nil
}
// CondensedRows counts distinct (reduced_fingerprint, minute) in both reduced tables, since the condensing views append each minute several times.
// A minute never appears in both tables: gauges go to one and counters to the other.
// Dimensions come from the reduced series; a condensed row without its series is still counted, without dimensions.
func (store *clickhouseStore) CondensedRows(ctx context.Context, dimensions []string, hour time.Time) ([]condensedmetertypes.Row, error) {
aliases := make([]string, len(dimensions))
selects := make([]string, len(dimensions))
for i, dimension := range dimensions {
aliases[i] = clickhousesql.Identifier(fmt.Sprintf("dimension_%d", i))
selects[i] = fmt.Sprintf("any(resource_attrs[%s]) AS %s", clickhousesql.StringLiteral(dimension), aliases[i])
}
condensed := func(table string) string {
return fmt.Sprintf("SELECT reduced_fingerprint, unix_milli FROM %s.%s WHERE unix_milli >= ? AND unix_milli < ? AND NOT match(metric_name, '^(signoz|otelcol)') GROUP BY reduced_fingerprint, unix_milli", metricstelemetryschema.DBName, table)
}
query := fmt.Sprintf(
"SELECT %s FROM (%s UNION ALL %s) AS condensed LEFT JOIN (SELECT %s FROM %s.%s WHERE unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint) AS series ON condensed.reduced_fingerprint = series.fingerprint",
strings.Join(append(aliases, "toUInt64(count())"), ", "),
condensed(metricstelemetryschema.SamplesV4ReducedLastTableName),
condensed(metricstelemetryschema.SamplesV4ReducedSumTableName),
strings.Join(append([]string{"fingerprint"}, selects...), ", "),
metricstelemetryschema.DBName,
metricstelemetryschema.TimeseriesV4ReducedTableName,
)
if len(aliases) > 0 {
query += " GROUP BY " + strings.Join(aliases, ", ")
}
start, end := hour.UnixMilli(), hour.Add(time.Hour).UnixMilli()
rows, err := store.telemetryStore.ClickhouseDB().Query(ctx, query, start, end, start, end, start, end)
if err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "failed to query condensed rows")
}
defer rows.Close()
condensedRows := []condensedmetertypes.Row{}
for rows.Next() {
row := condensedmetertypes.Row{Values: make([]string, len(dimensions))}
dest := make([]any, 0, len(dimensions)+1)
for i := range row.Values {
dest = append(dest, &row.Values[i])
}
dest = append(dest, &row.Count)
if err := rows.Scan(dest...); err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "failed to scan condensed rows")
}
if row.Count > 0 {
condensedRows = append(condensedRows, row)
}
}
return condensedRows, rows.Err()
}

View File

@@ -0,0 +1,215 @@
package kafkacondensedmeter
import (
"context"
"log/slog"
"time"
"github.com/twmb/franz-go/pkg/kgo"
"github.com/twmb/franz-go/plugin/kslog"
"go.opentelemetry.io/collector/pdata/pmetric"
"github.com/SigNoz/signoz/pkg/condensedmeter"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/licensing"
"github.com/SigNoz/signoz/pkg/modules/organization"
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/condensedmetertypes"
"github.com/SigNoz/signoz/pkg/types/featuretypes"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/SigNoz/signoz/pkg/version"
)
var _ condensedmeter.Reporter = (*provider)(nil)
type provider struct {
settings factory.ScopedProviderSettings
config condensedmeter.Config
flagger flagger.Flagger
licensing licensing.Licensing
orgGetter organization.Getter
clickhouseStore *clickhouseStore
store condensedmetertypes.Store
build version.Build
marshaler pmetric.ProtoMarshaler
client *kgo.Client
healthyC chan struct{}
stopC chan struct{}
}
func NewFactory(flagger flagger.Flagger, licensing licensing.Licensing, orgGetter organization.Getter, telemetryStore telemetrystore.TelemetryStore, sqlstore sqlstore.SQLStore, build version.Build) factory.ProviderFactory[condensedmeter.Reporter, condensedmeter.Config] {
return factory.NewProviderFactory(factory.MustNewName("kafka"), func(ctx context.Context, providerSettings factory.ProviderSettings, config condensedmeter.Config) (condensedmeter.Reporter, error) {
return newProvider(ctx, providerSettings, config, flagger, licensing, orgGetter, telemetryStore, newStore(sqlstore), build)
})
}
func newProvider(_ context.Context, providerSettings factory.ProviderSettings, config condensedmeter.Config, flagger flagger.Flagger, licensing licensing.Licensing, orgGetter organization.Getter, telemetryStore telemetrystore.TelemetryStore, store condensedmetertypes.Store, build version.Build) (*provider, error) {
settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/ee/condensedmeter/kafkacondensedmeter")
// Acks from all in-sync replicas, idempotent writes and unbounded record retries are franz-go defaults and are kept as is.
// A delivery timeout or retry limit lets franz-go < v1.22.2 ack a record that was never written (twmb/franz-go#1484).
client, err := kgo.NewClient(
kgo.SeedBrokers(config.Kafka.Brokers...),
kgo.ClientID(config.Kafka.ClientID),
kgo.ProducerBatchCompression(compressionCodec(config.Kafka.Compression)),
kgo.ProducerLinger(0),
kgo.WithLogger(kslog.New(settings.Logger())),
)
if err != nil {
return nil, err
}
return &provider{
settings: settings,
config: config,
flagger: flagger,
licensing: licensing,
orgGetter: orgGetter,
clickhouseStore: newClickhouseStore(telemetryStore),
store: store,
build: build,
marshaler: pmetric.ProtoMarshaler{},
client: client,
healthyC: make(chan struct{}),
stopC: make(chan struct{}),
}, nil
}
func (provider *provider) Start(ctx context.Context) error {
close(provider.healthyC)
ticker := time.NewTicker(provider.config.Interval)
defer ticker.Stop()
provider.report(ctx)
for {
select {
case <-provider.stopC:
return nil
case <-ticker.C:
provider.report(ctx)
}
}
}
func (provider *provider) Stop(_ context.Context) error {
close(provider.stopC)
provider.client.Close()
return nil
}
func (provider *provider) Healthy() <-chan struct{} {
return provider.healthyC
}
func (provider *provider) report(ctx context.Context) {
orgs, err := provider.orgGetter.ListByOwnedKeyRange(ctx)
if err != nil {
provider.settings.Logger().ErrorContext(ctx, "failed to get orgs data", errors.Attr(err))
return
}
for _, org := range orgs {
if !provider.flagger.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(org.ID)) {
continue
}
if _, err := provider.licensing.GetActive(ctx, org.ID); err != nil {
if errors.Ast(err, errors.TypeNotFound) {
continue
}
provider.settings.Logger().ErrorContext(ctx, "failed to fetch active license for org", errors.Attr(err), slog.String("org_id", org.ID.StringValue()))
return
}
if err := provider.reportOrg(ctx, org.ID, time.Now()); err != nil {
provider.settings.Logger().ErrorContext(ctx, "failed to report condensed rows", errors.Attr(err), slog.String("org_id", org.ID.StringValue()))
}
}
}
func (provider *provider) reportOrg(ctx context.Context, orgID valuer.UUID, now time.Time) error {
next, err := provider.nextHour(ctx, orgID)
if err != nil || next.IsZero() {
return err
}
for _, hour := range condensedmetertypes.ClosedHours(next, now) {
rows, err := provider.clickhouseStore.CondensedRows(ctx, provider.config.Dimensions, hour)
if err != nil {
return err
}
if len(rows) > 0 {
if err := provider.send(ctx, rows, now); err != nil {
return err
}
}
if err := provider.store.Upsert(ctx, condensedmetertypes.NewCheckpoint(orgID, hour)); err != nil {
return err
}
}
return nil
}
// nextHour is the hour after the checkpoint, or the first hour after the first ingested meter row; zero when there is none yet.
func (provider *provider) nextHour(ctx context.Context, orgID valuer.UUID) (time.Time, error) {
checkpoint, err := provider.store.Get(ctx, orgID)
if err == nil {
return checkpoint.LastReportedHour.UTC().Add(time.Hour), nil
}
if !errors.Ast(err, errors.TypeNotFound) {
return time.Time{}, err
}
firstIngested, err := provider.clickhouseStore.FirstIngested(ctx)
if err != nil || firstIngested.IsZero() {
return time.Time{}, err
}
return condensedmetertypes.FirstHour(firstIngested), nil
}
// send stamps the points with the send time, not the hour they count, so an hour reported after its day is sealed lands in the next day instead of being missed.
func (provider *provider) send(ctx context.Context, rows []condensedmetertypes.Row, timestamp time.Time) error {
batchSize := provider.config.BatchSize
records := make([]*kgo.Record, 0, (len(rows)+batchSize-1)/batchSize)
for start := 0; start < len(rows); start += batchSize {
metrics := condensedmetertypes.NewPMetrics(rows[start:min(start+batchSize, len(rows))], provider.config.Dimensions, timestamp, "signoz", provider.build.Version())
value, err := provider.marshaler.MarshalMetrics(metrics)
if err != nil {
return errors.WrapInternalf(err, errors.CodeInternal, "failed to marshal condensed rows")
}
records = append(records, &kgo.Record{Topic: provider.config.Kafka.Topic, Value: value})
}
if err := provider.client.ProduceSync(ctx, records...).FirstErr(); err != nil {
return errors.WrapInternalf(err, errors.CodeInternal, "failed to send condensed rows")
}
return nil
}
func compressionCodec(name string) kgo.CompressionCodec {
switch name {
case "gzip":
return kgo.GzipCompression()
case "snappy":
return kgo.SnappyCompression()
case "lz4":
return kgo.Lz4Compression()
case "zstd":
return kgo.ZstdCompression()
default:
return kgo.NoCompression()
}
}

View File

@@ -0,0 +1,176 @@
package kafkacondensedmeter
import (
"context"
"regexp"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
cmock "github.com/SigNoz/clickhouse-go-mock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/twmb/franz-go/pkg/kfake"
"github.com/twmb/franz-go/pkg/kgo"
"go.opentelemetry.io/collector/pdata/pmetric"
"github.com/SigNoz/signoz/pkg/condensedmeter"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
"github.com/SigNoz/signoz/pkg/types/condensedmetertypes"
"github.com/SigNoz/signoz/pkg/types/condensedmetertypes/condensedmetertypestest"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/SigNoz/signoz/pkg/version"
)
// Polls the topic until it has wantRecords records or the timeout passes.
func consumeMetrics(t *testing.T, brokers []string, topic string, wantRecords int, timeout time.Duration) []pmetric.Metrics {
t.Helper()
client, err := kgo.NewClient(
kgo.SeedBrokers(brokers...),
kgo.ConsumeTopics(topic),
kgo.ConsumeResetOffset(kgo.NewOffset().AtStart()),
)
require.NoError(t, err)
defer client.Close()
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
metrics := []pmetric.Metrics{}
for len(metrics) < wantRecords && ctx.Err() == nil {
for _, record := range client.PollFetches(ctx).Records() {
recordMetrics, err := (&pmetric.ProtoUnmarshaler{}).UnmarshalMetrics(record.Value)
require.NoError(t, err)
metrics = append(metrics, recordMetrics)
}
}
return metrics
}
func TestReportOrg(t *testing.T) {
orgID := valuer.GenerateUUID()
now := time.Date(2026, 11, 12, 13, 30, 0, 0, time.UTC)
checkpointHour := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
// 11:00 is closed at 13:30 (11:00 + 2h = 13:00); 12:00 is not (14:00)
reportedHour := time.Date(2026, 11, 12, 11, 0, 0, 0, time.UTC)
notFound := errors.New(errors.TypeNotFound, condensedmetertypes.ErrCodeCheckpointNotFound, "no checkpoint")
condensedColumns := []cmock.ColumnType{{Name: "dimension_0", Type: "String"}, {Name: "dimension_1", Type: "String"}, {Name: "count", Type: "UInt64"}}
testCases := []struct {
name string
checkpoint *condensedmetertypes.Checkpoint
firstIngestedMs int64
condensedRows [][]any
condensedErr error
expectedUpsert *time.Time
expectedRecords int
expectedDataPoint []int64
expectedErr bool
}{
{
name: "NoCheckpointNoIngestedRows_NothingReported",
firstIngestedMs: 0,
},
{
name: "Checkpoint_ClosedHourSentAndCheckpointAdvanced",
checkpoint: condensedmetertypes.NewCheckpoint(orgID, checkpointHour),
condensedRows: [][]any{{"frontend", "host-1", uint64(31)}, {"checkout", "", uint64(4)}},
expectedUpsert: &reportedHour,
expectedRecords: 1,
expectedDataPoint: []int64{31, 4},
},
{
name: "Checkpoint_LargeHourSplitAcrossRecords",
checkpoint: condensedmetertypes.NewCheckpoint(orgID, checkpointHour),
condensedRows: [][]any{{"frontend", "host-1", uint64(31)}, {"checkout", "", uint64(4)}, {"cart", "host-2", uint64(9)}},
expectedUpsert: &reportedHour,
expectedRecords: 2,
expectedDataPoint: []int64{31, 4, 9},
},
{
name: "Checkpoint_EmptyHourNotSentCheckpointAdvanced",
checkpoint: condensedmetertypes.NewCheckpoint(orgID, checkpointHour),
condensedRows: [][]any{},
expectedUpsert: &reportedHour,
},
{
name: "Checkpoint_QueryFails_CheckpointNotAdvanced",
checkpoint: condensedmetertypes.NewCheckpoint(orgID, checkpointHour),
condensedErr: errors.New(errors.TypeInternal, errors.CodeInternal, "clickhouse down"),
expectedErr: true,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
cluster, err := kfake.NewCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, "tenant_meter"))
require.NoError(t, err)
defer cluster.Close()
telemetryStore := telemetrystoretest.New(telemetrystore.Config{}, sqlmock.QueryMatcherRegexp)
store := condensedmetertypestest.NewMockStore(t)
if testCase.checkpoint != nil {
store.EXPECT().Get(mock.Anything, orgID).Return(testCase.checkpoint, nil)
} else {
store.EXPECT().Get(mock.Anything, orgID).Return(nil, notFound)
telemetryStore.Mock().ExpectQueryRow(regexp.QuoteMeta("min(unix_milli)")).
WillReturnRow(cmock.NewRow([]cmock.ColumnType{{Name: "min", Type: "Int64"}}, []any{testCase.firstIngestedMs}))
}
if testCase.condensedRows != nil || testCase.condensedErr != nil {
query := telemetryStore.Mock().ExpectQuery(regexp.QuoteMeta("distributed_time_series_v4_reduced")).
WithArgs(reportedHour.UnixMilli(), reportedHour.Add(time.Hour).UnixMilli(), reportedHour.UnixMilli(), reportedHour.Add(time.Hour).UnixMilli(), reportedHour.UnixMilli(), reportedHour.Add(time.Hour).UnixMilli())
if testCase.condensedErr != nil {
query.WillReturnError(testCase.condensedErr)
} else {
query.WillReturnRows(cmock.NewRows(condensedColumns, testCase.condensedRows))
}
}
if testCase.expectedUpsert != nil {
store.EXPECT().Upsert(mock.Anything, mock.MatchedBy(func(checkpoint *condensedmetertypes.Checkpoint) bool {
return checkpoint.OrgID == orgID && checkpoint.LastReportedHour.Equal(*testCase.expectedUpsert)
})).Return(nil)
}
config := condensedmeter.Config{
Provider: "kafka",
Interval: time.Hour,
Dimensions: []string{"service.name", "host.name"},
BatchSize: 2,
Kafka: condensedmeter.KafkaConfig{Brokers: cluster.ListenAddrs(), Topic: "tenant_meter", ClientID: "signoz-test", Compression: "none"},
}
provider, err := newProvider(context.Background(), instrumentationtest.New().ToProviderSettings(), config, nil, nil, nil, telemetryStore, store, version.Info)
require.NoError(t, err)
defer provider.client.Close()
err = provider.reportOrg(context.Background(), orgID, now)
if testCase.expectedErr {
assert.Error(t, err)
} else {
require.NoError(t, err)
}
assert.NoError(t, telemetryStore.Mock().ExpectationsWereMet())
metrics := consumeMetrics(t, cluster.ListenAddrs(), "tenant_meter", testCase.expectedRecords+1, 500*time.Millisecond)
require.Len(t, metrics, testCase.expectedRecords)
dataPoints := []int64{}
for _, recordMetrics := range metrics {
points := recordMetrics.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints()
for i := 0; i < points.Len(); i++ {
dataPoints = append(dataPoints, points.At(i).IntValue())
assert.Equal(t, now, points.At(i).Timestamp().AsTime())
}
}
assert.ElementsMatch(t, testCase.expectedDataPoint, dataPoints)
})
}
}

View File

@@ -0,0 +1,43 @@
package kafkacondensedmeter
import (
"context"
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/SigNoz/signoz/pkg/types/condensedmetertypes"
"github.com/SigNoz/signoz/pkg/valuer"
)
type store struct {
sqlstore sqlstore.SQLStore
}
func newStore(sqlstore sqlstore.SQLStore) condensedmetertypes.Store {
return &store{sqlstore: sqlstore}
}
func (store *store) Get(ctx context.Context, orgID valuer.UUID) (*condensedmetertypes.Checkpoint, error) {
checkpoint := new(condensedmetertypes.Checkpoint)
err := store.sqlstore.
BunDBCtx(ctx).
NewSelect().
Model(checkpoint).
Where("org_id = ?", orgID).
Scan(ctx)
if err != nil {
return nil, store.sqlstore.WrapNotFoundErrf(err, condensedmetertypes.ErrCodeCheckpointNotFound, "no condensed meter checkpoint found for org %q", orgID.StringValue())
}
return checkpoint, nil
}
func (store *store) Upsert(ctx context.Context, checkpoint *condensedmetertypes.Checkpoint) error {
_, err := store.sqlstore.
BunDBCtx(ctx).
NewInsert().
Model(checkpoint).
On("CONFLICT (org_id) DO UPDATE").
Set("last_reported_hour = EXCLUDED.last_reported_hour").
Set("updated_at = EXCLUDED.updated_at").
Exec(ctx)
return err
}

View File

@@ -0,0 +1,14 @@
package condensedmeter
import (
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
)
var (
ErrCodeInvalidInput = errors.MustNewCode("condensedmeter_invalid_input")
)
type Reporter interface {
factory.ServiceWithHealthy
}

View File

@@ -0,0 +1,85 @@
package condensedmeter
import (
"time"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
)
var _ factory.Config = (*Config)(nil)
type Config struct {
Provider string `mapstructure:"provider"`
Interval time.Duration `mapstructure:"interval"`
// Dimensions are the resource attributes the meter connector records as meter labels.
// They must match the connector's dimensions, so condensed rows slice like every other meter row.
Dimensions []string `mapstructure:"dimensions"`
// BatchSize is the maximum number of data points per Kafka record.
BatchSize int `mapstructure:"batch_size"`
Kafka KafkaConfig `mapstructure:"kafka"`
}
type KafkaConfig struct {
// Brokers are the seed broker addresses (host:port) used to discover the cluster.
Brokers []string `mapstructure:"brokers"`
// Topic must already exist; the producer does not create it.
Topic string `mapstructure:"topic"`
// ClientID identifies this producer in broker logs and metrics.
ClientID string `mapstructure:"client_id"`
// Compression is the codec for produced batches: none, gzip, snappy, lz4 or zstd.
Compression string `mapstructure:"compression"`
}
func newConfig() factory.Config {
return Config{
Provider: "noop",
Interval: time.Hour,
Dimensions: []string{"service.name", "deployment.environment", "host.name", "signoz.workspace.key.id"},
BatchSize: 100,
Kafka: KafkaConfig{
Brokers: []string{"localhost:9092"},
ClientID: "signoz",
Compression: "lz4",
},
}
}
func NewConfigFactory() factory.ConfigFactory {
return factory.NewConfigFactory(factory.MustNewName("condensedmeter"), newConfig)
}
func (c Config) Validate() error {
if c.Interval < 10*time.Minute || c.Interval > time.Hour {
return errors.New(errors.TypeInvalidInput, ErrCodeInvalidInput, "condensedmeter::interval must be between 10m and 1h")
}
if c.BatchSize <= 0 {
return errors.New(errors.TypeInvalidInput, ErrCodeInvalidInput, "condensedmeter::batch_size must be greater than 0")
}
if c.Provider == "kafka" {
if len(c.Kafka.Brokers) == 0 {
return errors.New(errors.TypeInvalidInput, ErrCodeInvalidInput, "condensedmeter::kafka::brokers must be set when provider is kafka")
}
if c.Kafka.Topic == "" {
return errors.New(errors.TypeInvalidInput, ErrCodeInvalidInput, "condensedmeter::kafka::topic must be set when provider is kafka")
}
switch c.Kafka.Compression {
case "none", "gzip", "snappy", "lz4", "zstd":
default:
return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidInput, "condensedmeter::kafka::compression must be one of none, gzip, snappy, lz4 or zstd, got %q", c.Kafka.Compression)
}
}
return nil
}

View File

@@ -0,0 +1,39 @@
package noopcondensedmeter
import (
"context"
"github.com/SigNoz/signoz/pkg/condensedmeter"
"github.com/SigNoz/signoz/pkg/factory"
)
type provider struct {
healthyC chan struct{}
stopC chan struct{}
}
func NewFactory() factory.ProviderFactory[condensedmeter.Reporter, condensedmeter.Config] {
return factory.NewProviderFactory(factory.MustNewName("noop"), New)
}
func New(_ context.Context, _ factory.ProviderSettings, _ condensedmeter.Config) (condensedmeter.Reporter, error) {
return &provider{
healthyC: make(chan struct{}),
stopC: make(chan struct{}),
}, nil
}
func (provider *provider) Start(_ context.Context) error {
close(provider.healthyC)
<-provider.stopC
return nil
}
func (provider *provider) Stop(_ context.Context) error {
close(provider.stopC)
return nil
}
func (provider *provider) Healthy() <-chan struct{} {
return provider.healthyC
}

View File

@@ -26,9 +26,8 @@ const bucketCacheCellSteps = 60
type BucketCache interface {
// cached portion + list of gaps to fetch
GetMissRanges(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step) (cached *qbtypes.Result, missing []*qbtypes.TimeRange)
// store fresh buckets for future hits; only the ranges in the map were
// queried for fresh, the rest of it came from the cache
Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats)
// store fresh buckets for future hits
Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result)
}
// bucketCache implements the BucketCache interface.
@@ -117,7 +116,7 @@ func (bc *bucketCache) GetMissRanges(
}
// Merge buckets into a single result
mergedResult := bc.mergeBuckets(ctx, relevantBuckets, cachedWarnings, startMs, endMs)
mergedResult := bc.mergeBuckets(ctx, relevantBuckets, cachedWarnings)
// Filter the merged result to only include values within the requested time range
mergedResult = bc.filterResultToTimeRange(mergedResult, q, startMs, endMs, stepMs)
@@ -126,7 +125,7 @@ func (bc *bucketCache) GetMissRanges(
}
// Put stores fresh query results in the cache.
func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats) {
func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result) {
// Get query window
startMs, endMs := q.Window()
@@ -189,7 +188,7 @@ func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Que
// Convert trimmed result to buckets, one per grid cell it spans
cellWidthMs := bucketCacheCellSteps * stepMs
freshBuckets := bc.resultToBuckets(ctx, trimmedResult, cachableStartMs, cachableEndMs, cellWidthMs, execStatsForQueriedTimeRanges)
freshBuckets := bc.resultToBuckets(ctx, trimmedResult, cachableStartMs, cachableEndMs, cellWidthMs)
cellKeys := make([]string, len(freshBuckets))
cellEntries := make([]cachetypes.Cacheable, len(freshBuckets))
@@ -211,7 +210,7 @@ func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Que
cellEntry = cellEntries[i].(*qbtypes.CachedData)
}
cellBuckets, changed := bc.mergeFreshBucketIntoCell(ctx, cellEntry.Buckets, freshBucket, execStatsForQueriedTimeRanges)
cellBuckets, changed := bc.mergeFreshBucketIntoCell(ctx, cellEntry.Buckets, freshBucket)
if !changed {
continue
}
@@ -464,7 +463,7 @@ func (bc *bucketCache) filterRelevantBuckets(buckets []*qbtypes.CachedBucket, st
}
// mergeBuckets combines multiple cached buckets into a single result.
func (bc *bucketCache) mergeBuckets(ctx context.Context, buckets []*qbtypes.CachedBucket, warnings []string, startMs, endMs uint64) *qbtypes.Result {
func (bc *bucketCache) mergeBuckets(ctx context.Context, buckets []*qbtypes.CachedBucket, warnings []string) *qbtypes.Result {
if len(buckets) == 0 {
return &qbtypes.Result{}
}
@@ -472,13 +471,12 @@ func (bc *bucketCache) mergeBuckets(ctx context.Context, buckets []*qbtypes.Cach
// All buckets should have the same type
resultType := buckets[0].Type
// A bucket can reach past the window; only the part inside it is served
// Aggregate stats
var totalStats qbtypes.ExecStats
for _, bucket := range buckets {
servedStats := bucket.ProrateStatsToWindow(startMs, endMs)
totalStats.RowsScanned += servedStats.RowsScanned
totalStats.BytesScanned += servedStats.BytesScanned
totalStats.DurationMS += servedStats.DurationMS
totalStats.RowsScanned += bucket.Stats.RowsScanned
totalStats.BytesScanned += bucket.Stats.BytesScanned
totalStats.DurationMS += bucket.Stats.DurationMS
}
// Merge values based on type
@@ -675,7 +673,7 @@ func (bc *bucketCache) isEmptyResult(result *qbtypes.Result) (isEmpty bool, isFi
}
// resultToBuckets converts a query result into time-based buckets.
func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Result, startMs, endMs, cellWidthMs uint64, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats) []*qbtypes.CachedBucket {
func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Result, startMs, endMs, cellWidthMs uint64) []*qbtypes.CachedBucket {
// Check if result is empty
isEmpty, isFiltered := bc.isEmptyResult(result)
@@ -687,7 +685,7 @@ func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Resu
}
// Empty filtered results still become buckets, so the range is not re-queried
perCellBuckets, err := result.SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs, execStatsForQueriedTimeRanges)
perCellBuckets, err := result.SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs)
if err != nil {
bc.logger.ErrorContext(ctx, "failed to convert result to buckets", errors.Attr(err))
return nil
@@ -699,7 +697,7 @@ func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Resu
// and reports whether the cell changed: a bucket already covering the fresh
// one leaves it as is, buckets inside the fresh one are replaced, buckets
// overlapping or edge to edge with it are unioned, and the rest are kept apart.
func (bc *bucketCache) mergeFreshBucketIntoCell(ctx context.Context, cellBuckets []*qbtypes.CachedBucket, freshBucket *qbtypes.CachedBucket, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats) ([]*qbtypes.CachedBucket, bool) {
func (bc *bucketCache) mergeFreshBucketIntoCell(ctx context.Context, cellBuckets []*qbtypes.CachedBucket, freshBucket *qbtypes.CachedBucket) ([]*qbtypes.CachedBucket, bool) {
keptBuckets := make([]*qbtypes.CachedBucket, 0, len(cellBuckets)+1)
touchingBuckets := []*qbtypes.CachedBucket{}
for _, cellBucket := range cellBuckets {
@@ -712,7 +710,7 @@ func (bc *bucketCache) mergeFreshBucketIntoCell(ctx context.Context, cellBuckets
keptBuckets = append(keptBuckets, cellBucket)
}
}
keptBuckets = append(keptBuckets, bc.unionBuckets(ctx, freshBucket, touchingBuckets, execStatsForQueriedTimeRanges))
keptBuckets = append(keptBuckets, bc.unionBuckets(ctx, freshBucket, touchingBuckets))
slices.SortFunc(keptBuckets, func(a, b *qbtypes.CachedBucket) int {
return cmp.Compare(a.StartMs, b.StartMs)
@@ -723,16 +721,7 @@ func (bc *bucketCache) mergeFreshBucketIntoCell(ctx context.Context, cellBuckets
// unionBuckets widens the fresh bucket by the buckets it touches. Buckets
// inside it are replaced outright, so only a genuine partial overlap decodes
// anything.
func (bc *bucketCache) unionBuckets(ctx context.Context, freshBucket *qbtypes.CachedBucket, touchingBuckets []*qbtypes.CachedBucket, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats) *qbtypes.CachedBucket {
// Data queried again takes the new query's cost, which the fresh bucket
// carries; a touching bucket keeps its cost only for what was not queried again
for _, touchingBucket := range touchingBuckets {
keptStats := touchingBucket.ProrateStatsToUnqueried(execStatsForQueriedTimeRanges)
freshBucket.Stats.RowsScanned += keptStats.RowsScanned
freshBucket.Stats.BytesScanned += keptStats.BytesScanned
freshBucket.Stats.DurationMS += keptStats.DurationMS
}
func (bc *bucketCache) unionBuckets(ctx context.Context, freshBucket *qbtypes.CachedBucket, touchingBuckets []*qbtypes.CachedBucket) *qbtypes.CachedBucket {
bucketsToMerge := []*qbtypes.CachedBucket{freshBucket}
for _, touchingBucket := range touchingBuckets {
if !freshBucket.Contains(touchingBucket) {
@@ -753,11 +742,14 @@ func (bc *bucketCache) unionBuckets(ctx context.Context, freshBucket *qbtypes.Ca
EndMs: freshBucket.EndMs,
Type: freshBucket.Type,
Value: value,
Stats: freshBucket.Stats,
Stats: qbtypes.ExecStats{StepIntervals: freshBucket.Stats.StepIntervals},
}
for _, mergedBucket := range bucketsToMerge {
unionBucket.StartMs = min(unionBucket.StartMs, mergedBucket.StartMs)
unionBucket.EndMs = max(unionBucket.EndMs, mergedBucket.EndMs)
unionBucket.Stats.RowsScanned += mergedBucket.Stats.RowsScanned
unionBucket.Stats.BytesScanned += mergedBucket.Stats.BytesScanned
unionBucket.Stats.DurationMS += mergedBucket.Stats.DurationMS
}
return unionBucket
}

View File

@@ -30,7 +30,7 @@ func BenchmarkBucketCache_GetMissRanges(b *testing.B) {
endMs: uint64((i + 1) * 10000),
}
result := createBenchmarkResult(query.startMs, query.endMs, 1000)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
}
// Create test queries with varying cache hit patterns
@@ -121,7 +121,7 @@ func BenchmarkBucketCache_Put(b *testing.B) {
for i := 0; i < b.N; i++ {
for j := 0; j < tc.numQueries; j++ {
bc.Put(ctx, orgID, queries[j], qbtypes.Step{Duration: 1000 * time.Millisecond}, results[j], map[qbtypes.TimeRange]qbtypes.ExecStats{{From: queries[j].startMs, To: queries[j].endMs}: results[j].Stats})
bc.Put(ctx, orgID, queries[j], qbtypes.Step{Duration: 1000 * time.Millisecond}, results[j])
}
}
})
@@ -259,7 +259,7 @@ func BenchmarkBucketCache_ConcurrentOperations(b *testing.B) {
endMs: uint64((i + 1) * 10000),
}
result := createBenchmarkResult(query.startMs, query.endMs, 1000)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
}
b.ResetTimer()
@@ -284,7 +284,7 @@ func BenchmarkBucketCache_ConcurrentOperations(b *testing.B) {
endMs: uint64((i + 1) * 10000),
}
result := createBenchmarkResult(query.startMs, query.endMs, 1000)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
case 2: // Partial read
query := &mockQuery{
fingerprint: fmt.Sprintf("concurrent-query-%d", i%100),

View File

@@ -59,7 +59,7 @@ func TestBucketCacheStepAlignment(t *testing.T) {
}
// Put result in cache
bc.Put(ctx, orgID, query, step, result, nil)
bc.Put(ctx, orgID, query, step, result)
// Get cached data
cached, missing := bc.GetMissRanges(ctx, orgID, query, step)
@@ -109,7 +109,7 @@ func TestBucketCacheNoStepInterval(t *testing.T) {
}
// Should cache the entire range when step is 0
bc.Put(ctx, orgID, query, step, result, nil)
bc.Put(ctx, orgID, query, step, result)
cached, missing := bc.GetMissRanges(ctx, orgID, query, step)
assert.NotNil(t, cached)

View File

@@ -156,7 +156,7 @@ func TestBucketCache_Put_And_Get(t *testing.T) {
}
// Store in cache
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
// Wait a bit for cache to be written
time.Sleep(10 * time.Millisecond)
@@ -189,7 +189,7 @@ func TestBucketCache_PartialHit(t *testing.T) {
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 1000, 3000, 1000),
}
bc.Put(context.Background(), valuer.UUID{}, query1, qbtypes.Step{Duration: 1000 * time.Millisecond}, result1, nil)
bc.Put(context.Background(), valuer.UUID{}, query1, qbtypes.Step{Duration: 1000 * time.Millisecond}, result1)
// Wait for cache write
time.Sleep(10 * time.Millisecond)
@@ -225,7 +225,7 @@ func TestBucketCache_MultipleBuckets(t *testing.T) {
bc.Put(context.Background(), valuer.UUID{}, query1, qbtypes.Step{Duration: 100 * time.Millisecond}, &qbtypes.Result{
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 1000, 2000, 100),
}, nil)
})
query2 := &mockQuery{
fingerprint: "test-query",
@@ -235,7 +235,7 @@ func TestBucketCache_MultipleBuckets(t *testing.T) {
bc.Put(context.Background(), valuer.UUID{}, query2, qbtypes.Step{Duration: 100 * time.Millisecond}, &qbtypes.Result{
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 3000, 4000, 100),
}, nil)
})
// Wait for cache writes
time.Sleep(10 * time.Millisecond)
@@ -280,7 +280,7 @@ func TestBucketCache_FluxInterval(t *testing.T) {
}
// This should not be cached due to flux interval
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
// Wait a bit
time.Sleep(10 * time.Millisecond)
@@ -358,7 +358,7 @@ func TestBucketCache_MergeTimeSeriesResults(t *testing.T) {
{Series: []*qbtypes.TimeSeries{series1}},
},
},
}, nil)
})
// Cache second part
query2 := &mockQuery{
@@ -374,7 +374,7 @@ func TestBucketCache_MergeTimeSeriesResults(t *testing.T) {
{Series: []*qbtypes.TimeSeries{series2}},
},
},
}, nil)
})
// Wait for cache writes
time.Sleep(10 * time.Millisecond)
@@ -441,7 +441,7 @@ func TestBucketCache_RawData(t *testing.T) {
Value: rawData,
}
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
time.Sleep(10 * time.Millisecond)
cached, missing := bc.GetMissRanges(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
@@ -481,7 +481,7 @@ func TestBucketCache_ScalarData(t *testing.T) {
Value: scalarData,
}
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
time.Sleep(10 * time.Millisecond)
cached, missing := bc.GetMissRanges(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
@@ -509,7 +509,7 @@ func TestBucketCache_EmptyFingerprint(t *testing.T) {
Value: createTestTimeSeries("A", 1000, 5000, 1000),
}
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
time.Sleep(10 * time.Millisecond)
// Should still be able to retrieve
@@ -564,7 +564,7 @@ func TestBucketCache_ConcurrentAccess(t *testing.T) {
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries(fmt.Sprintf("Q%d", id), query.startMs, query.endMs, 100),
}
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 100 * time.Microsecond}, result, nil)
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 100 * time.Microsecond}, result)
done <- true
}(i)
}
@@ -624,7 +624,7 @@ func TestBucketCache_GetMissRanges_FluxInterval(t *testing.T) {
},
}
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, cachedResult, nil)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, cachedResult)
// Get miss ranges
cached, missing := bc.GetMissRanges(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
@@ -686,7 +686,7 @@ func TestBucketCache_Put_FluxIntervalTrimming(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
// Retrieve cached data
cached, missing := bc.GetMissRanges(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
@@ -756,7 +756,7 @@ func TestBucketCache_Put_EntireRangeInFluxInterval(t *testing.T) {
}
// Put the result - should not cache anything
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
// Try to get cached data - should have no cached data
cached, missing := bc.GetMissRanges(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
@@ -862,7 +862,7 @@ func TestBucketCache_EmptyDataHandling(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, tt.result, nil)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, tt.result)
// Wait a bit for cache to be written
time.Sleep(10 * time.Millisecond)
@@ -927,7 +927,7 @@ func TestBucketCache_PartialValues(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
// Wait for cache to be written
time.Sleep(10 * time.Millisecond)
@@ -997,7 +997,7 @@ func TestBucketCache_AllPartialValues(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
// Wait for cache to be written
time.Sleep(10 * time.Millisecond)
@@ -1058,7 +1058,7 @@ func TestBucketCache_FilteredCachedResults(t *testing.T) {
}
// Cache the wide range
bc.Put(ctx, orgID, query1, qbtypes.Step{Duration: 1000 * time.Millisecond}, result1, nil)
bc.Put(ctx, orgID, query1, qbtypes.Step{Duration: 1000 * time.Millisecond}, result1)
time.Sleep(10 * time.Millisecond)
// Now query for a smaller range (2000-3500ms)
@@ -1125,7 +1125,7 @@ func TestBucketCache_PromQLKeepsTheValueAtTheWindowEnd(t *testing.T) {
}},
}},
},
}, nil)
})
time.Sleep(10 * time.Millisecond)
cached, missing := bc.GetMissRanges(ctx, orgID, query, step)
@@ -1291,7 +1291,7 @@ func TestBucketCache_PartialValueDetection(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
time.Sleep(10 * time.Millisecond)
// Get cached data
@@ -1345,7 +1345,7 @@ func TestBucketCache_PartialValueDetection(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
time.Sleep(10 * time.Millisecond)
// Get cached data
@@ -1397,7 +1397,7 @@ func TestBucketCache_PartialValueDetection(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
time.Sleep(10 * time.Millisecond)
// Get cached data
@@ -1454,7 +1454,7 @@ func TestBucketCache_NoCache(t *testing.T) {
}
// Put the result in cache
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
time.Sleep(10 * time.Millisecond)
// Verify data is cached
@@ -1475,12 +1475,11 @@ func TestBucketCachePutSlicesAtCellBoundaries(t *testing.T) {
// a 1s step gives 60s cells, so 30s to 150s spans three of them
query := &mockQuery{fingerprint: "sliced-query", startMs: 30000, endMs: 150000}
result := &qbtypes.Result{
bc.Put(ctx, orgID, query, step, &qbtypes.Result{
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 30000, 150000, 1000),
Stats: qbtypes.ExecStats{RowsScanned: 120, BytesScanned: 1200, DurationMS: 12},
}
bc.Put(ctx, orgID, query, step, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: 30000, To: 150000}: result.Stats})
})
type bucketShape struct {
cell, startMs, endMs, rowsScanned uint64
@@ -1540,7 +1539,7 @@ func TestBucketCachePutAbsorbsWritesWithinACell(t *testing.T) {
name: "OverlappingWrites_BecomeOne",
firstWindow: qbtypes.TimeRange{From: 1000, To: 4000},
secondWindow: qbtypes.TimeRange{From: 3000, To: 6000},
expectedBuckets: []bucketShape{{1000, 6000, 5}},
expectedBuckets: []bucketShape{{1000, 6000, 6}},
expectedMissingRanges: []*qbtypes.TimeRange{},
expectedValueCount: 5,
},
@@ -1571,13 +1570,12 @@ func TestBucketCachePutAbsorbsWritesWithinACell(t *testing.T) {
for _, window := range []qbtypes.TimeRange{testCase.firstWindow, testCase.secondWindow} {
query := &mockQuery{fingerprint: "absorb-query", startMs: window.From, endMs: window.To}
result := &qbtypes.Result{
bc.Put(ctx, orgID, query, step, &qbtypes.Result{
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", window.From, window.To, 1000),
// one row per second, so a bucket's stats read as the seconds of data it holds
// one row per second, so a bucket's stats read as the seconds it was scanned for
Stats: qbtypes.ExecStats{RowsScanned: (window.To - window.From) / 1000},
}
bc.Put(ctx, orgID, query, step, result, map[qbtypes.TimeRange]qbtypes.ExecStats{window: result.Stats})
})
}
query := &mockQuery{
@@ -1611,7 +1609,7 @@ func TestBucketCachePutLeavesAnUnchangedCellAlone(t *testing.T) {
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 1000, 10000, 1000),
Warnings: []string{"from the wide write"},
}, nil)
})
// a write already covered by the cell's bucket must not rewrite the entry,
// which shows as its warnings never arriving
@@ -1620,7 +1618,7 @@ func TestBucketCachePutLeavesAnUnchangedCellAlone(t *testing.T) {
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 3000, 5000, 1000),
Warnings: []string{"from the inside write"},
}, nil)
})
var cellEntry qbtypes.CachedData
require.NoError(t, bc.cache.Get(ctx, orgID, bc.generateCacheKey(wide, 0), &cellEntry))

View File

@@ -828,7 +828,7 @@ func (q *querier) executeWithCache(ctx context.Context, orgID valuer.UUID, query
return nil, err
}
// Store in cache for future use
q.bucketCache.Put(ctx, orgID, query, step, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: startMs, To: endMs}: result.Stats})
q.bucketCache.Put(ctx, orgID, query, step, result)
return result, nil
}
}
@@ -884,19 +884,16 @@ func (q *querier) executeWithCache(ctx context.Context, orgID valuer.UUID, query
if err != nil {
return nil, err
}
startMs, endMs := query.Window()
q.bucketCache.Put(ctx, orgID, query, step, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: startMs, To: endMs}: result.Stats})
q.bucketCache.Put(ctx, orgID, query, step, result)
return result, nil
}
}
// Calculate total stats and filter out nil results
validResults := make([]*qbtypes.Result, 0, len(freshResults))
execStatsForQueriedTimeRanges := make(map[qbtypes.TimeRange]qbtypes.ExecStats, len(freshResults))
for i, result := range freshResults {
for _, result := range freshResults {
if result != nil {
validResults = append(validResults, result)
execStatsForQueriedTimeRanges[*missingRanges[i]] = result.Stats
totalStats.RowsScanned += result.Stats.RowsScanned
totalStats.BytesScanned += result.Stats.BytesScanned
totalStats.DurationMS += result.Stats.DurationMS
@@ -911,7 +908,7 @@ func (q *querier) executeWithCache(ctx context.Context, orgID valuer.UUID, query
mergedResult.Stats.DurationMS += totalStats.DurationMS
// Store merged result in cache
q.bucketCache.Put(ctx, orgID, query, step, mergedResult, execStatsForQueriedTimeRanges)
q.bucketCache.Put(ctx, orgID, query, step, mergedResult)
return mergedResult, nil
}

View File

@@ -14,6 +14,7 @@ import (
"github.com/SigNoz/signoz/pkg/auditor"
"github.com/SigNoz/signoz/pkg/authz"
"github.com/SigNoz/signoz/pkg/cache"
"github.com/SigNoz/signoz/pkg/condensedmeter"
"github.com/SigNoz/signoz/pkg/config"
"github.com/SigNoz/signoz/pkg/emailing"
"github.com/SigNoz/signoz/pkg/errors"
@@ -139,6 +140,9 @@ type Config struct {
// MeterReporter config
MeterReporter meterreporter.Config `mapstructure:"meterreporter"`
// CondensedMeter config
CondensedMeter condensedmeter.Config `mapstructure:"condensedmeter"`
// CloudIntegration config
CloudIntegration cloudintegration.Config `mapstructure:"cloudintegration"`
@@ -180,6 +184,7 @@ func NewConfig(ctx context.Context, logger *slog.Logger, resolverConfig config.R
serviceaccount.NewConfigFactory(),
auditor.NewConfigFactory(),
meterreporter.NewConfigFactory(),
condensedmeter.NewConfigFactory(),
cloudintegration.NewConfigFactory(),
tracedetail.NewConfigFactory(),
authz.NewConfigFactory(),

View File

@@ -16,6 +16,8 @@ import (
"github.com/SigNoz/signoz/pkg/cache"
"github.com/SigNoz/signoz/pkg/cache/memorycache"
"github.com/SigNoz/signoz/pkg/cache/rediscache"
"github.com/SigNoz/signoz/pkg/condensedmeter"
"github.com/SigNoz/signoz/pkg/condensedmeter/noopcondensedmeter"
"github.com/SigNoz/signoz/pkg/emailing"
"github.com/SigNoz/signoz/pkg/emailing/noopemailing"
"github.com/SigNoz/signoz/pkg/emailing/smtpemailing"
@@ -275,6 +277,7 @@ func NewSQLMigrationProviderFactories(
sqlmigration.NewMigrateRulesV1ToV2Alpha1Factory(sqlstore),
sqlmigration.NewAddLLMPricingRuleTuplesFactory(sqlstore),
sqlmigration.NewRemoveUnsupportedQuickFiltersFactory(),
sqlmigration.NewAddCondensedMeterCheckpointFactory(sqlstore, sqlschema),
)
}
@@ -429,6 +432,12 @@ func NewMeterReporterProviderFactories() factory.NamedMap[factory.ProviderFactor
)
}
func NewCondensedMeterProviderFactories() factory.NamedMap[factory.ProviderFactory[condensedmeter.Reporter, condensedmeter.Config]] {
return factory.MustNewNamedMap(
noopcondensedmeter.NewFactory(),
)
}
func NewFlaggerProviderFactories(registry featuretypes.Registry) factory.NamedMap[factory.ProviderFactory[flagger.FlaggerProvider, flagger.Config]] {
return factory.MustNewNamedMap(
configflagger.NewFactory(registry),

View File

@@ -15,6 +15,7 @@ import (
"github.com/SigNoz/signoz/pkg/authn/authnstore/sqlauthnstore"
"github.com/SigNoz/signoz/pkg/authz"
"github.com/SigNoz/signoz/pkg/cache"
"github.com/SigNoz/signoz/pkg/condensedmeter"
"github.com/SigNoz/signoz/pkg/emailing"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
@@ -99,6 +100,7 @@ type SigNoz struct {
Gateway gateway.Gateway
Auditor auditor.Auditor
MeterReporter meterreporter.Reporter
CondensedMeter condensedmeter.Reporter
}
// newQueryStack assembles the query stack once and returns, in order: the shared
@@ -184,6 +186,7 @@ func New(
gatewayProviderFactory func(licensing.Licensing) factory.ProviderFactory[gateway.Gateway, gateway.Config],
auditorProviderFactories func(licensing.Licensing) factory.NamedMap[factory.ProviderFactory[auditor.Auditor, auditor.Config]],
meterReporterProviderFactories func(context.Context, factory.ProviderSettings, flagger.Flagger, licensing.Licensing, telemetrystore.TelemetryStore, retention.Getter, organization.Getter, zeus.Zeus) (factory.NamedMap[factory.ProviderFactory[meterreporter.Reporter, meterreporter.Config]], string),
condensedMeterProviderFactories func(flagger.Flagger, licensing.Licensing, organization.Getter, telemetrystore.TelemetryStore, sqlstore.SQLStore) factory.NamedMap[factory.ProviderFactory[condensedmeter.Reporter, condensedmeter.Config]],
querierHandlerCallback func(factory.ProviderSettings, querier.Querier, analytics.Analytics) querier.Handler,
cloudIntegrationCallback func(sqlstore.SQLStore, dashboard.Module, global.Global, zeus.Zeus, gateway.Gateway, licensing.Licensing, serviceaccount.Module, cloudintegration.Config) (cloudintegration.Module, error),
metricReductionRuleModuleCallback func(sqlstore.SQLStore, telemetrystore.TelemetryStore, dashboard.Module, queryparser.QueryParser, licensing.Licensing, flagger.Flagger, telemetrytypes.MetadataStore, factory.ProviderSettings, int) metricreductionrule.Module,
@@ -501,6 +504,12 @@ func New(
return nil, err
}
// Initialize condensed meter from the variant-specific provider factories
condensedMeter, err := factory.NewProviderFromNamedMap(ctx, providerSettings, config.CondensedMeter, condensedMeterProviderFactories(flagger, licensing, orgGetter, telemetrystore, sqlstore), config.CondensedMeter.Provider)
if err != nil {
return nil, err
}
// Initialize authns
store := sqlauthnstore.NewStore(sqlstore)
authNs, err := authNsCallback(ctx, providerSettings, store, licensing)
@@ -606,6 +615,7 @@ func New(
factory.NewNamedService(factory.MustNewName("user"), userService, factory.MustNewName("authz")),
factory.NewNamedService(factory.MustNewName("auditor"), auditor),
factory.NewNamedService(factory.MustNewName("meterreporter"), meterReporter, factory.MustNewName("licensing")),
factory.NewNamedService(factory.MustNewName("condensedmeter"), condensedMeter, factory.MustNewName("licensing")),
factory.NewNamedService(factory.MustNewName("ruler"), rulerInstance),
factory.NewNamedService(factory.MustNewName("systemdashboard"), impldashboard.NewService(providerSettings, dashboard, orgGetter)),
factory.NewNamedService(factory.MustNewName("spanmappergroup"), implspanmapper.NewService(providerSettings, spanMapperModule, orgGetter)),
@@ -667,5 +677,6 @@ func New(
Gateway: gateway,
Auditor: auditor,
MeterReporter: meterReporter,
CondensedMeter: condensedMeter,
}, nil
}

View File

@@ -0,0 +1,78 @@
package sqlmigration
import (
"context"
"github.com/uptrace/bun"
"github.com/uptrace/bun/migrate"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/sqlschema"
"github.com/SigNoz/signoz/pkg/sqlstore"
)
type addCondensedMeterCheckpoint struct {
sqlschema sqlschema.SQLSchema
sqlstore sqlstore.SQLStore
}
func NewAddCondensedMeterCheckpointFactory(sqlstore sqlstore.SQLStore, sqlschema sqlschema.SQLSchema) factory.ProviderFactory[SQLMigration, Config] {
return factory.NewProviderFactory(factory.MustNewName("add_condensed_meter_checkpoint"), func(_ context.Context, _ factory.ProviderSettings, _ Config) (SQLMigration, error) {
return &addCondensedMeterCheckpoint{
sqlschema: sqlschema,
sqlstore: sqlstore,
}, nil
})
}
func (migration *addCondensedMeterCheckpoint) Register(migrations *migrate.Migrations) error {
if err := migrations.Register(migration.Up, migration.Down); err != nil {
return err
}
return nil
}
func (migration *addCondensedMeterCheckpoint) Up(ctx context.Context, db *bun.DB) error {
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
sqls := [][]byte{}
tableSQLs := migration.sqlschema.Operator().CreateTable(&sqlschema.Table{
Name: "condensed_meter_checkpoint",
Columns: []*sqlschema.Column{
{Name: "org_id", DataType: sqlschema.DataTypeText, Nullable: false},
{Name: "last_reported_hour", DataType: sqlschema.DataTypeTimestamp, Nullable: false},
{Name: "updated_at", DataType: sqlschema.DataTypeTimestamp, Nullable: false},
},
PrimaryKeyConstraint: &sqlschema.PrimaryKeyConstraint{
ColumnNames: []sqlschema.ColumnName{"org_id"},
},
ForeignKeyConstraints: []*sqlschema.ForeignKeyConstraint{
{
ReferencingColumnName: sqlschema.ColumnName("org_id"),
ReferencedTableName: sqlschema.TableName("organizations"),
ReferencedColumnName: sqlschema.ColumnName("id"),
},
},
})
sqls = append(sqls, tableSQLs...)
for _, sql := range sqls {
if _, err := tx.ExecContext(ctx, string(sql)); err != nil {
return err
}
}
return tx.Commit()
}
func (migration *addCondensedMeterCheckpoint) Down(context.Context, *bun.DB) error {
return nil
}

View File

@@ -0,0 +1,36 @@
package condensedmetertypes
import (
"context"
"time"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/uptrace/bun"
)
var (
ErrCodeCheckpointNotFound = errors.MustNewCode("condensed_meter_checkpoint_not_found")
)
// Checkpoint is the last hour whose condensed rows were sent for an org.
type Checkpoint struct {
bun.BaseModel `bun:"table:condensed_meter_checkpoint"`
OrgID valuer.UUID `bun:"org_id,pk,type:text"`
LastReportedHour time.Time `bun:"last_reported_hour"`
UpdatedAt time.Time `bun:"updated_at"`
}
type Store interface {
Get(ctx context.Context, orgID valuer.UUID) (*Checkpoint, error)
Upsert(ctx context.Context, checkpoint *Checkpoint) error
}
func NewCheckpoint(orgID valuer.UUID, hour time.Time) *Checkpoint {
return &Checkpoint{
OrgID: orgID,
LastReportedHour: hour,
UpdatedAt: time.Now(),
}
}

View File

@@ -0,0 +1,165 @@
// Code generated by mockery; DO NOT EDIT.
// github.com/vektra/mockery
// template: testify
package condensedmetertypestest
import (
"context"
"github.com/SigNoz/signoz/pkg/types/condensedmetertypes"
"github.com/SigNoz/signoz/pkg/valuer"
mock "github.com/stretchr/testify/mock"
)
// NewMockStore creates a new instance of MockStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockStore(t interface {
mock.TestingT
Cleanup(func())
}) *MockStore {
mock := &MockStore{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
// MockStore is an autogenerated mock type for the Store type
type MockStore struct {
mock.Mock
}
type MockStore_Expecter struct {
mock *mock.Mock
}
func (_m *MockStore) EXPECT() *MockStore_Expecter {
return &MockStore_Expecter{mock: &_m.Mock}
}
// Get provides a mock function for the type MockStore
func (_mock *MockStore) Get(ctx context.Context, orgID valuer.UUID) (*condensedmetertypes.Checkpoint, error) {
ret := _mock.Called(ctx, orgID)
if len(ret) == 0 {
panic("no return value specified for Get")
}
var r0 *condensedmetertypes.Checkpoint
var r1 error
if returnFunc, ok := ret.Get(0).(func(context.Context, valuer.UUID) (*condensedmetertypes.Checkpoint, error)); ok {
return returnFunc(ctx, orgID)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, valuer.UUID) *condensedmetertypes.Checkpoint); ok {
r0 = returnFunc(ctx, orgID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*condensedmetertypes.Checkpoint)
}
}
if returnFunc, ok := ret.Get(1).(func(context.Context, valuer.UUID) error); ok {
r1 = returnFunc(ctx, orgID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockStore_Get_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Get'
type MockStore_Get_Call struct {
*mock.Call
}
// Get is a helper method to define mock.On call
// - ctx context.Context
// - orgID valuer.UUID
func (_e *MockStore_Expecter) Get(ctx interface{}, orgID interface{}) *MockStore_Get_Call {
return &MockStore_Get_Call{Call: _e.mock.On("Get", ctx, orgID)}
}
func (_c *MockStore_Get_Call) Run(run func(ctx context.Context, orgID valuer.UUID)) *MockStore_Get_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 valuer.UUID
if args[1] != nil {
arg1 = args[1].(valuer.UUID)
}
run(
arg0,
arg1,
)
})
return _c
}
func (_c *MockStore_Get_Call) Return(checkpoint *condensedmetertypes.Checkpoint, err error) *MockStore_Get_Call {
_c.Call.Return(checkpoint, err)
return _c
}
func (_c *MockStore_Get_Call) RunAndReturn(run func(ctx context.Context, orgID valuer.UUID) (*condensedmetertypes.Checkpoint, error)) *MockStore_Get_Call {
_c.Call.Return(run)
return _c
}
// Upsert provides a mock function for the type MockStore
func (_mock *MockStore) Upsert(ctx context.Context, checkpoint *condensedmetertypes.Checkpoint) error {
ret := _mock.Called(ctx, checkpoint)
if len(ret) == 0 {
panic("no return value specified for Upsert")
}
var r0 error
if returnFunc, ok := ret.Get(0).(func(context.Context, *condensedmetertypes.Checkpoint) error); ok {
r0 = returnFunc(ctx, checkpoint)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockStore_Upsert_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Upsert'
type MockStore_Upsert_Call struct {
*mock.Call
}
// Upsert is a helper method to define mock.On call
// - ctx context.Context
// - checkpoint *condensedmetertypes.Checkpoint
func (_e *MockStore_Expecter) Upsert(ctx interface{}, checkpoint interface{}) *MockStore_Upsert_Call {
return &MockStore_Upsert_Call{Call: _e.mock.On("Upsert", ctx, checkpoint)}
}
func (_c *MockStore_Upsert_Call) Run(run func(ctx context.Context, checkpoint *condensedmetertypes.Checkpoint)) *MockStore_Upsert_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 *condensedmetertypes.Checkpoint
if args[1] != nil {
arg1 = args[1].(*condensedmetertypes.Checkpoint)
}
run(
arg0,
arg1,
)
})
return _c
}
func (_c *MockStore_Upsert_Call) Return(err error) *MockStore_Upsert_Call {
_c.Call.Return(err)
return _c
}
func (_c *MockStore_Upsert_Call) RunAndReturn(run func(ctx context.Context, checkpoint *condensedmetertypes.Checkpoint) error) *MockStore_Upsert_Call {
_c.Call.Return(run)
return _c
}

View File

@@ -0,0 +1,22 @@
package condensedmetertypes
import "time"
// CondensingDelay is how long after an hour starts its condensed rows are final.
// Condensing of a minute finishes about 10 minutes after it, so the last minute of an hour is final within 2h of its start.
const CondensingDelay = 2 * time.Hour
// FirstHour is the hour after the first ingested meter row. Before it, datapoint.count meant ingested, so adding condensed rows would count them twice.
func FirstHour(firstIngested time.Time) time.Time {
return firstIngested.UTC().Truncate(time.Hour).Add(time.Hour)
}
// ClosedHours returns the hours from next whose condensed rows are final at now, oldest first.
func ClosedHours(next time.Time, now time.Time) []time.Time {
hours := []time.Time{}
for hour := next; !hour.Add(CondensingDelay).After(now); hour = hour.Add(time.Hour) {
hours = append(hours, hour)
}
return hours
}

View File

@@ -0,0 +1,48 @@
package condensedmetertypes
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestFirstHour(t *testing.T) {
testCases := []struct {
name string
firstIngested time.Time
expected time.Time
}{
{name: "MidHour_NextHour", firstIngested: time.Date(2026, 11, 12, 9, 39, 38, 0, time.UTC), expected: time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)},
{name: "OnTheHour_NextHour", firstIngested: time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC), expected: time.Date(2026, 11, 12, 11, 0, 0, 0, time.UTC)},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
assert.Equal(t, testCase.expected, FirstHour(testCase.firstIngested))
})
}
}
func TestClosedHours(t *testing.T) {
next := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
testCases := []struct {
name string
now time.Time
expected []time.Time
}{
// 10:00 + 2h = 12:00 is after 11:59
{name: "NotClosed_None", now: time.Date(2026, 11, 12, 11, 59, 0, 0, time.UTC), expected: []time.Time{}},
// 10:00 + 2h = 12:00
{name: "ClosedAtDelay_One", now: time.Date(2026, 11, 12, 12, 0, 0, 0, time.UTC), expected: []time.Time{next}},
// 12:00 + 2h = 14:00 is before 14:30, 13:00 + 2h = 15:00 is not
{name: "Backlog_OldestFirst", now: time.Date(2026, 11, 12, 14, 30, 0, 0, time.UTC), expected: []time.Time{next, next.Add(time.Hour), next.Add(2 * time.Hour)}},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
assert.Equal(t, testCase.expected, ClosedHours(next, testCase.now))
})
}
}

View File

@@ -0,0 +1,49 @@
package condensedmetertypes
import (
"time"
"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/pmetric"
)
const (
metricNameMetricsDataPointsCount = "signoz.meter.metric.datapoint.count"
metricDescMetricsDataPointsCount = "The number of data points indexed."
)
// Row is the number of condensed rows of one dimension set in one hour.
// Values are positional, in the order of the configured dimensions; an empty value means the dimension is absent.
type Row struct {
Values []string
Count uint64
}
// NewPMetrics shapes rows like the meter connector's output (dimensions as data point attributes, delta monotonic sum), so they land in the same meter table and slice the same way.
func NewPMetrics(rows []Row, dimensions []string, timestamp time.Time, scopeName string, scopeVersion string) pmetric.Metrics {
metrics := pmetric.NewMetrics()
scopeMetrics := metrics.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty()
scopeMetrics.Scope().SetName(scopeName)
scopeMetrics.Scope().SetVersion(scopeVersion)
metric := scopeMetrics.Metrics().AppendEmpty()
metric.SetName(metricNameMetricsDataPointsCount)
metric.SetDescription(metricDescMetricsDataPointsCount)
metric.SetUnit("1")
sum := metric.SetEmptySum()
sum.SetAggregationTemporality(pmetric.AggregationTemporalityDelta)
sum.SetIsMonotonic(true)
for _, row := range rows {
dataPoint := sum.DataPoints().AppendEmpty()
dataPoint.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
dataPoint.SetIntValue(int64(row.Count))
for i, dimension := range dimensions {
if row.Values[i] != "" {
dataPoint.Attributes().PutStr(dimension, row.Values[i])
}
}
}
return metrics
}

View File

@@ -0,0 +1,57 @@
package condensedmetertypes
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/collector/pdata/pmetric"
)
func TestNewPMetrics(t *testing.T) {
timestamp := time.Date(2026, 11, 12, 13, 5, 0, 0, time.UTC)
dimensions := []string{"service.name", "host.name"}
rows := []Row{
{Values: []string{"frontend", "host-1"}, Count: 31},
{Values: []string{"checkout", ""}, Count: 4},
}
metrics := NewPMetrics(rows, dimensions, timestamp, "signoz", "v0.0.1")
require.Equal(t, 1, metrics.ResourceMetrics().Len())
resourceMetrics := metrics.ResourceMetrics().At(0)
assert.Equal(t, 0, resourceMetrics.Resource().Attributes().Len())
require.Equal(t, 1, resourceMetrics.ScopeMetrics().Len())
scopeMetrics := resourceMetrics.ScopeMetrics().At(0)
assert.Equal(t, "signoz", scopeMetrics.Scope().Name())
assert.Equal(t, "v0.0.1", scopeMetrics.Scope().Version())
require.Equal(t, 1, scopeMetrics.Metrics().Len())
metric := scopeMetrics.Metrics().At(0)
assert.Equal(t, "signoz.meter.metric.datapoint.count", metric.Name())
require.Equal(t, pmetric.MetricTypeSum, metric.Type())
assert.Equal(t, pmetric.AggregationTemporalityDelta, metric.Sum().AggregationTemporality())
assert.True(t, metric.Sum().IsMonotonic())
dataPoints := metric.Sum().DataPoints()
require.Equal(t, 2, dataPoints.Len())
testCases := []struct {
name string
index int
value int64
attributes map[string]any
}{
{name: "AllDimensions", index: 0, value: 31, attributes: map[string]any{"service.name": "frontend", "host.name": "host-1"}},
{name: "EmptyDimension_Omitted", index: 1, value: 4, attributes: map[string]any{"service.name": "checkout"}},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
dataPoint := dataPoints.At(testCase.index)
assert.Equal(t, testCase.value, dataPoint.IntValue())
assert.Equal(t, timestamp, dataPoint.Timestamp().AsTime())
assert.Equal(t, testCase.attributes, dataPoint.Attributes().AsRaw())
})
}
}

View File

@@ -43,23 +43,6 @@ func (c *CachedBucket) Touches(other *CachedBucket) bool {
return c.StartMs <= other.EndMs && other.StartMs <= c.EndMs
}
// ProrateStatsToWindow returns the share of the stats for the part of the
// bucket inside [startMs, endMs).
func (c *CachedBucket) ProrateStatsToWindow(startMs, endMs uint64) ExecStats {
return c.Stats.prorate(overlapMs(c.StartMs, c.EndMs, startMs, endMs), c.EndMs-c.StartMs)
}
// ProrateStatsToUnqueried returns the share of the stats for the part of the
// bucket no queried range covers.
func (c *CachedBucket) ProrateStatsToUnqueried(execStatsForQueriedTimeRanges map[TimeRange]ExecStats) ExecStats {
widthMs := c.EndMs - c.StartMs
queriedMs := uint64(0)
for queried := range execStatsForQueriedTimeRanges {
queriedMs += overlapMs(c.StartMs, c.EndMs, queried.From, queried.To)
}
return c.Stats.prorate(widthMs-min(queriedMs, widthMs), widthMs)
}
// CachedData represents the full cached data for a query.
type CachedData struct {
Buckets []*CachedBucket `json:"buckets"`
@@ -105,7 +88,7 @@ func (c *CachedData) Cost() int64 {
return size
}
func (r *Result) SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs uint64, execStatsForQueriedTimeRanges map[TimeRange]ExecStats) ([]*CachedBucket, error) {
func (r *Result) SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs uint64) ([]*CachedBucket, error) {
tsData, ok := r.Value.(*TimeSeriesData)
if !ok || tsData == nil {
return nil, errors.NewInternalf(errors.CodeInternal, "result of type %s cannot be bucketed", r.Type.StringValue())
@@ -128,39 +111,18 @@ func (r *Result) SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs uint64, e
if err != nil {
return nil, err
}
// A slice costs what was queried inside it; the rest of the result came
// out of the cache and is costed there already
sliceStats := ExecStats{StepIntervals: maps.Clone(r.Stats.StepIntervals)}
for queried, stats := range execStatsForQueriedTimeRanges {
queriedMs := overlapMs(from, to, queried.From, queried.To)
if queriedMs == 0 {
continue
}
share := stats.prorate(queriedMs, queried.To-queried.From)
sliceStats.RowsScanned += share.RowsScanned
sliceStats.BytesScanned += share.BytesScanned
sliceStats.DurationMS += share.DurationMS
}
cellBuckets = append(cellBuckets, &CachedBucket{
StartMs: from,
EndMs: to,
Type: r.Type,
Value: value,
Stats: sliceStats,
Stats: r.Stats.prorate(to-from, endMs-startMs),
})
from = to
}
return cellBuckets, nil
}
// overlapMs is the length of [aFromMs, aToMs) ∩ [bFromMs, bToMs).
func overlapMs(aFromMs, aToMs, bFromMs, bToMs uint64) uint64 {
if fromMs, toMs := max(aFromMs, bFromMs), min(aToMs, bToMs); fromMs < toMs {
return toMs - fromMs
}
return 0
}
// prorate returns the share of the stats that partMs out of wholeMs of the
// scanned window accounts for.
func (e ExecStats) prorate(partMs, wholeMs uint64) ExecStats {

View File

@@ -442,55 +442,3 @@ def test_builder_refreshing_a_sliding_time_range(
assert uncached.status_code == HTTPStatus.OK, uncached.text
assert_results_equal(from_cache.json(), uncached.json(), "A", "the twentieth refresh")
def test_builder_refreshing_does_not_inflate_rows_scanned(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_metrics: Callable[[list[Metrics]], None],
) -> None:
metric_name = f"cache_rows_scanned_{uuid4().hex[:8]}"
# 90 minutes back so even the last refresh closes clear of the flux
# interval, which holds recent data out of the cache
start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=90)).timestamp()) // 60 * 60, tz=UTC)
start_time_ms = int(start_time.timestamp() * 1000)
query = [build_builder_query("A", metric_name, "max", "max")]
insert_metrics(
[
Metrics(
metric_name=metric_name,
labels={"service": "api"},
timestamp=start_time + timedelta(minutes=minute),
value=minute,
type_="Gauge",
is_monotonic=False,
)
for minute in range(70)
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# a dashboard left open on a thirty minute range, re-running a minute later
# each time. rowsScanned is a cost indicator, so a refresh served mostly out
# of the cache cannot claim more than every read made so far: the first
# full read and then one new minute per refresh, measured with the cache off
uncached_rows_scanned_so_far = 0
for refresh in range(10):
refresh_start_ms = start_time_ms + refresh * MINUTE_MS
refresh_end_ms = refresh_start_ms + 30 * MINUTE_MS
uncached_start_ms = refresh_start_ms if refresh == 0 else refresh_end_ms - MINUTE_MS
uncached_response = make_query_request(signoz, token, uncached_start_ms, refresh_end_ms, query, no_cache=True)
assert uncached_response.status_code == HTTPStatus.OK, f"refresh {refresh}: uncached read failed: {uncached_response.text}"
uncached_rows_scanned_so_far += uncached_response.json()["data"]["meta"]["rowsScanned"]
assert uncached_rows_scanned_so_far > 0, f"refresh {refresh}: uncached read reported no rows scanned"
cached_response = make_query_request(signoz, token, refresh_start_ms, refresh_end_ms, query, no_cache=False)
assert cached_response.status_code == HTTPStatus.OK, f"refresh {refresh}: cached read failed: {cached_response.text}"
cached_rows_scanned = cached_response.json()["data"]["meta"]["rowsScanned"]
assert cached_rows_scanned <= uncached_rows_scanned_so_far, f"refresh {refresh}: cached read reported {cached_rows_scanned} rows scanned, more than the {uncached_rows_scanned_so_far} rows all uncached reads scanned so far"