mirror of
https://github.com/SigNoz/signoz.git
synced 2026-11-16 13:21:15 +00:00
Compare commits
7 Commits
nv/exec-st
...
feat/conde
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c7892db471 | ||
|
|
59b8295da0 | ||
|
|
7360d79258 | ||
|
|
9aa7226130 | ||
|
|
573f197161 | ||
|
|
702919dd00 | ||
|
|
aff6346cc5 |
@@ -16,6 +16,14 @@ packages:
|
||||
filename: "maintenance.go"
|
||||
structname: 'Mock{{.InterfaceName}}'
|
||||
pkgname: '{{.SrcPackageName}}test'
|
||||
github.com/SigNoz/signoz/pkg/types/condensedmetertypes:
|
||||
interfaces:
|
||||
Store:
|
||||
config:
|
||||
dir: '{{.InterfaceDir}}/condensedmetertypestest'
|
||||
filename: "store.go"
|
||||
structname: 'Mock{{.InterfaceName}}'
|
||||
pkgname: '{{.SrcPackageName}}test'
|
||||
github.com/SigNoz/signoz/pkg/types/telemetrytypes:
|
||||
interfaces:
|
||||
SchemaMetadataStore:
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/authz/openfgaschema"
|
||||
"github.com/SigNoz/signoz/pkg/authz/openfgaserver"
|
||||
"github.com/SigNoz/signoz/pkg/cache"
|
||||
"github.com/SigNoz/signoz/pkg/condensedmeter"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
@@ -121,6 +122,9 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
|
||||
func(_ context.Context, _ factory.ProviderSettings, _ flagger.Flagger, _ licensing.Licensing, _ telemetrystore.TelemetryStore, _ retention.Getter, _ organization.Getter, _ zeus.Zeus) (factory.NamedMap[factory.ProviderFactory[meterreporter.Reporter, meterreporter.Config]], string) {
|
||||
return signoz.NewMeterReporterProviderFactories(), "noop"
|
||||
},
|
||||
func(_ flagger.Flagger, _ licensing.Licensing, _ organization.Getter, _ telemetrystore.TelemetryStore, _ sqlstore.SQLStore) factory.NamedMap[factory.ProviderFactory[condensedmeter.Reporter, condensedmeter.Config]] {
|
||||
return signoz.NewCondensedMeterProviderFactories()
|
||||
},
|
||||
func(ps factory.ProviderSettings, q querier.Querier, a analytics.Analytics) querier.Handler {
|
||||
return querier.NewHandler(ps, q, a)
|
||||
},
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/SigNoz/signoz/ee/authz/openfgaauthz"
|
||||
"github.com/SigNoz/signoz/ee/authz/openfgaschema"
|
||||
"github.com/SigNoz/signoz/ee/authz/openfgaserver"
|
||||
"github.com/SigNoz/signoz/ee/condensedmeter/kafkacondensedmeter"
|
||||
"github.com/SigNoz/signoz/ee/gateway/httpgateway"
|
||||
enterpriselicensing "github.com/SigNoz/signoz/ee/licensing"
|
||||
"github.com/SigNoz/signoz/ee/licensing/httplicensing"
|
||||
@@ -38,6 +39,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/authn"
|
||||
"github.com/SigNoz/signoz/pkg/authz"
|
||||
"github.com/SigNoz/signoz/pkg/cache"
|
||||
"github.com/SigNoz/signoz/pkg/condensedmeter"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
pkgflagger "github.com/SigNoz/signoz/pkg/flagger"
|
||||
@@ -176,6 +178,13 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
|
||||
|
||||
return factories, "http"
|
||||
},
|
||||
func(flagger pkgflagger.Flagger, licensing licensing.Licensing, orgGetter organization.Getter, telemetryStore telemetrystore.TelemetryStore, sqlstore sqlstore.SQLStore) factory.NamedMap[factory.ProviderFactory[condensedmeter.Reporter, condensedmeter.Config]] {
|
||||
factories := signoz.NewCondensedMeterProviderFactories()
|
||||
if err := factories.Add(kafkacondensedmeter.NewFactory(flagger, licensing, orgGetter, telemetryStore, sqlstore, version.Info)); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return factories
|
||||
},
|
||||
func(ps factory.ProviderSettings, q querier.Querier, a analytics.Analytics) querier.Handler {
|
||||
communityHandler := querier.NewHandler(ps, q, a)
|
||||
return eequerier.NewHandler(ps, q, communityHandler)
|
||||
|
||||
@@ -522,3 +522,30 @@ meterreporter:
|
||||
# interval - random(0, jitter). Must be between 10m and interval. Defaults to
|
||||
# min(interval, 2h) when unset.
|
||||
jitter: 2h
|
||||
|
||||
##################### Condensed Meter #####################
|
||||
condensedmeter:
|
||||
# Specifies the condensed meter provider to use.
|
||||
# noop: reports nothing (community default).
|
||||
# kafka: sends the hourly count of condensed metric rows to a Kafka topic as OTLP protobuf (enterprise).
|
||||
provider: noop
|
||||
# How often closed hours are looked for. Minimum 10m, maximum 1h.
|
||||
interval: 1h
|
||||
# The meter connector's dimensions. Condensed rows are split by them like every other meter row.
|
||||
dimensions:
|
||||
- service.name
|
||||
- deployment.environment
|
||||
- host.name
|
||||
- signoz.workspace.key.id
|
||||
# The maximum number of data points per Kafka record.
|
||||
batch_size: 100
|
||||
kafka:
|
||||
# The seed broker addresses (host:port) used to discover the cluster.
|
||||
brokers:
|
||||
- localhost:9092
|
||||
# The topic condensed rows are written to. It must already exist.
|
||||
topic: ""
|
||||
# The client id the producer reports to the brokers.
|
||||
client_id: signoz
|
||||
# The compression codec for produced batches: none, gzip, snappy, lz4 or zstd.
|
||||
compression: lz4
|
||||
|
||||
97
ee/condensedmeter/kafkacondensedmeter/clickhouse.go
Normal file
97
ee/condensedmeter/kafkacondensedmeter/clickhouse.go
Normal file
@@ -0,0 +1,97 @@
|
||||
package kafkacondensedmeter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/clickhousesql"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/condensedmetertypes"
|
||||
)
|
||||
|
||||
const metricNameMetricsDataPointsIngestedCount = "signoz.meter.metric.datapoint.ingested.count"
|
||||
|
||||
type clickhouseStore struct {
|
||||
telemetryStore telemetrystore.TelemetryStore
|
||||
}
|
||||
|
||||
func newClickhouseStore(telemetryStore telemetrystore.TelemetryStore) *clickhouseStore {
|
||||
return &clickhouseStore{telemetryStore: telemetryStore}
|
||||
}
|
||||
|
||||
// FirstIngested is the time of the first ingested meter row, zero when there is none yet.
|
||||
func (store *clickhouseStore) FirstIngested(ctx context.Context) (time.Time, error) {
|
||||
query := fmt.Sprintf("SELECT toInt64(ifNull(min(unix_milli), 0)) FROM %s.%s WHERE metric_name = ?", metertelemetryschema.DBName, metertelemetryschema.SamplesTableName)
|
||||
|
||||
var unixMilli int64
|
||||
if err := store.telemetryStore.ClickhouseDB().QueryRow(ctx, query, metricNameMetricsDataPointsIngestedCount).Scan(&unixMilli); err != nil {
|
||||
return time.Time{}, errors.WrapInternalf(err, errors.CodeInternal, "failed to query first ingested meter row")
|
||||
}
|
||||
|
||||
if unixMilli == 0 {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
|
||||
return time.UnixMilli(unixMilli), nil
|
||||
}
|
||||
|
||||
// CondensedRows counts distinct (reduced_fingerprint, minute) in both reduced tables, since the condensing views append each minute several times.
|
||||
// A minute never appears in both tables: gauges go to one and counters to the other.
|
||||
// Dimensions come from the reduced series; a condensed row without its series is still counted, without dimensions.
|
||||
func (store *clickhouseStore) CondensedRows(ctx context.Context, dimensions []string, hour time.Time) ([]condensedmetertypes.Row, error) {
|
||||
aliases := make([]string, len(dimensions))
|
||||
selects := make([]string, len(dimensions))
|
||||
for i, dimension := range dimensions {
|
||||
aliases[i] = clickhousesql.Identifier(fmt.Sprintf("dimension_%d", i))
|
||||
selects[i] = fmt.Sprintf("any(resource_attrs[%s]) AS %s", clickhousesql.StringLiteral(dimension), aliases[i])
|
||||
}
|
||||
|
||||
condensed := func(table string) string {
|
||||
return fmt.Sprintf("SELECT reduced_fingerprint, unix_milli FROM %s.%s WHERE unix_milli >= ? AND unix_milli < ? AND NOT match(metric_name, '^(signoz|otelcol)') GROUP BY reduced_fingerprint, unix_milli", metricstelemetryschema.DBName, table)
|
||||
}
|
||||
|
||||
query := fmt.Sprintf(
|
||||
"SELECT %s FROM (%s UNION ALL %s) AS condensed LEFT JOIN (SELECT %s FROM %s.%s WHERE unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint) AS series ON condensed.reduced_fingerprint = series.fingerprint",
|
||||
strings.Join(append(aliases, "toUInt64(count())"), ", "),
|
||||
condensed(metricstelemetryschema.SamplesV4ReducedLastTableName),
|
||||
condensed(metricstelemetryschema.SamplesV4ReducedSumTableName),
|
||||
strings.Join(append([]string{"fingerprint"}, selects...), ", "),
|
||||
metricstelemetryschema.DBName,
|
||||
metricstelemetryschema.TimeseriesV4ReducedTableName,
|
||||
)
|
||||
if len(aliases) > 0 {
|
||||
query += " GROUP BY " + strings.Join(aliases, ", ")
|
||||
}
|
||||
|
||||
start, end := hour.UnixMilli(), hour.Add(time.Hour).UnixMilli()
|
||||
rows, err := store.telemetryStore.ClickhouseDB().Query(ctx, query, start, end, start, end, start, end)
|
||||
if err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "failed to query condensed rows")
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
condensedRows := []condensedmetertypes.Row{}
|
||||
for rows.Next() {
|
||||
row := condensedmetertypes.Row{Values: make([]string, len(dimensions))}
|
||||
dest := make([]any, 0, len(dimensions)+1)
|
||||
for i := range row.Values {
|
||||
dest = append(dest, &row.Values[i])
|
||||
}
|
||||
dest = append(dest, &row.Count)
|
||||
|
||||
if err := rows.Scan(dest...); err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "failed to scan condensed rows")
|
||||
}
|
||||
|
||||
if row.Count > 0 {
|
||||
condensedRows = append(condensedRows, row)
|
||||
}
|
||||
}
|
||||
|
||||
return condensedRows, rows.Err()
|
||||
}
|
||||
215
ee/condensedmeter/kafkacondensedmeter/provider.go
Normal file
215
ee/condensedmeter/kafkacondensedmeter/provider.go
Normal file
@@ -0,0 +1,215 @@
|
||||
package kafkacondensedmeter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/twmb/franz-go/pkg/kgo"
|
||||
"github.com/twmb/franz-go/plugin/kslog"
|
||||
"go.opentelemetry.io/collector/pdata/pmetric"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/condensedmeter"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/licensing"
|
||||
"github.com/SigNoz/signoz/pkg/modules/organization"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/condensedmetertypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/SigNoz/signoz/pkg/version"
|
||||
)
|
||||
|
||||
var _ condensedmeter.Reporter = (*provider)(nil)
|
||||
|
||||
type provider struct {
|
||||
settings factory.ScopedProviderSettings
|
||||
config condensedmeter.Config
|
||||
flagger flagger.Flagger
|
||||
licensing licensing.Licensing
|
||||
orgGetter organization.Getter
|
||||
clickhouseStore *clickhouseStore
|
||||
store condensedmetertypes.Store
|
||||
build version.Build
|
||||
marshaler pmetric.ProtoMarshaler
|
||||
client *kgo.Client
|
||||
healthyC chan struct{}
|
||||
stopC chan struct{}
|
||||
}
|
||||
|
||||
func NewFactory(flagger flagger.Flagger, licensing licensing.Licensing, orgGetter organization.Getter, telemetryStore telemetrystore.TelemetryStore, sqlstore sqlstore.SQLStore, build version.Build) factory.ProviderFactory[condensedmeter.Reporter, condensedmeter.Config] {
|
||||
return factory.NewProviderFactory(factory.MustNewName("kafka"), func(ctx context.Context, providerSettings factory.ProviderSettings, config condensedmeter.Config) (condensedmeter.Reporter, error) {
|
||||
return newProvider(ctx, providerSettings, config, flagger, licensing, orgGetter, telemetryStore, newStore(sqlstore), build)
|
||||
})
|
||||
}
|
||||
|
||||
func newProvider(_ context.Context, providerSettings factory.ProviderSettings, config condensedmeter.Config, flagger flagger.Flagger, licensing licensing.Licensing, orgGetter organization.Getter, telemetryStore telemetrystore.TelemetryStore, store condensedmetertypes.Store, build version.Build) (*provider, error) {
|
||||
settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/ee/condensedmeter/kafkacondensedmeter")
|
||||
|
||||
// Acks from all in-sync replicas, idempotent writes and unbounded record retries are franz-go defaults and are kept as is.
|
||||
// A delivery timeout or retry limit lets franz-go < v1.22.2 ack a record that was never written (twmb/franz-go#1484).
|
||||
client, err := kgo.NewClient(
|
||||
kgo.SeedBrokers(config.Kafka.Brokers...),
|
||||
kgo.ClientID(config.Kafka.ClientID),
|
||||
kgo.ProducerBatchCompression(compressionCodec(config.Kafka.Compression)),
|
||||
kgo.ProducerLinger(0),
|
||||
kgo.WithLogger(kslog.New(settings.Logger())),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &provider{
|
||||
settings: settings,
|
||||
config: config,
|
||||
flagger: flagger,
|
||||
licensing: licensing,
|
||||
orgGetter: orgGetter,
|
||||
clickhouseStore: newClickhouseStore(telemetryStore),
|
||||
store: store,
|
||||
build: build,
|
||||
marshaler: pmetric.ProtoMarshaler{},
|
||||
client: client,
|
||||
healthyC: make(chan struct{}),
|
||||
stopC: make(chan struct{}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (provider *provider) Start(ctx context.Context) error {
|
||||
close(provider.healthyC)
|
||||
|
||||
ticker := time.NewTicker(provider.config.Interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
provider.report(ctx)
|
||||
for {
|
||||
select {
|
||||
case <-provider.stopC:
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
provider.report(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (provider *provider) Stop(_ context.Context) error {
|
||||
close(provider.stopC)
|
||||
provider.client.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (provider *provider) Healthy() <-chan struct{} {
|
||||
return provider.healthyC
|
||||
}
|
||||
|
||||
func (provider *provider) report(ctx context.Context) {
|
||||
orgs, err := provider.orgGetter.ListByOwnedKeyRange(ctx)
|
||||
if err != nil {
|
||||
provider.settings.Logger().ErrorContext(ctx, "failed to get orgs data", errors.Attr(err))
|
||||
return
|
||||
}
|
||||
|
||||
for _, org := range orgs {
|
||||
if !provider.flagger.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(org.ID)) {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err := provider.licensing.GetActive(ctx, org.ID); err != nil {
|
||||
if errors.Ast(err, errors.TypeNotFound) {
|
||||
continue
|
||||
}
|
||||
|
||||
provider.settings.Logger().ErrorContext(ctx, "failed to fetch active license for org", errors.Attr(err), slog.String("org_id", org.ID.StringValue()))
|
||||
return
|
||||
}
|
||||
|
||||
if err := provider.reportOrg(ctx, org.ID, time.Now()); err != nil {
|
||||
provider.settings.Logger().ErrorContext(ctx, "failed to report condensed rows", errors.Attr(err), slog.String("org_id", org.ID.StringValue()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (provider *provider) reportOrg(ctx context.Context, orgID valuer.UUID, now time.Time) error {
|
||||
next, err := provider.nextHour(ctx, orgID)
|
||||
if err != nil || next.IsZero() {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, hour := range condensedmetertypes.ClosedHours(next, now) {
|
||||
rows, err := provider.clickhouseStore.CondensedRows(ctx, provider.config.Dimensions, hour)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(rows) > 0 {
|
||||
if err := provider.send(ctx, rows, now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := provider.store.Upsert(ctx, condensedmetertypes.NewCheckpoint(orgID, hour)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// nextHour is the hour after the checkpoint, or the first hour after the first ingested meter row; zero when there is none yet.
|
||||
func (provider *provider) nextHour(ctx context.Context, orgID valuer.UUID) (time.Time, error) {
|
||||
checkpoint, err := provider.store.Get(ctx, orgID)
|
||||
if err == nil {
|
||||
return checkpoint.LastReportedHour.UTC().Add(time.Hour), nil
|
||||
}
|
||||
|
||||
if !errors.Ast(err, errors.TypeNotFound) {
|
||||
return time.Time{}, err
|
||||
}
|
||||
|
||||
firstIngested, err := provider.clickhouseStore.FirstIngested(ctx)
|
||||
if err != nil || firstIngested.IsZero() {
|
||||
return time.Time{}, err
|
||||
}
|
||||
|
||||
return condensedmetertypes.FirstHour(firstIngested), nil
|
||||
}
|
||||
|
||||
// send stamps the points with the send time, not the hour they count, so an hour reported after its day is sealed lands in the next day instead of being missed.
|
||||
func (provider *provider) send(ctx context.Context, rows []condensedmetertypes.Row, timestamp time.Time) error {
|
||||
batchSize := provider.config.BatchSize
|
||||
records := make([]*kgo.Record, 0, (len(rows)+batchSize-1)/batchSize)
|
||||
for start := 0; start < len(rows); start += batchSize {
|
||||
metrics := condensedmetertypes.NewPMetrics(rows[start:min(start+batchSize, len(rows))], provider.config.Dimensions, timestamp, "signoz", provider.build.Version())
|
||||
|
||||
value, err := provider.marshaler.MarshalMetrics(metrics)
|
||||
if err != nil {
|
||||
return errors.WrapInternalf(err, errors.CodeInternal, "failed to marshal condensed rows")
|
||||
}
|
||||
|
||||
records = append(records, &kgo.Record{Topic: provider.config.Kafka.Topic, Value: value})
|
||||
}
|
||||
|
||||
if err := provider.client.ProduceSync(ctx, records...).FirstErr(); err != nil {
|
||||
return errors.WrapInternalf(err, errors.CodeInternal, "failed to send condensed rows")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func compressionCodec(name string) kgo.CompressionCodec {
|
||||
switch name {
|
||||
case "gzip":
|
||||
return kgo.GzipCompression()
|
||||
case "snappy":
|
||||
return kgo.SnappyCompression()
|
||||
case "lz4":
|
||||
return kgo.Lz4Compression()
|
||||
case "zstd":
|
||||
return kgo.ZstdCompression()
|
||||
default:
|
||||
return kgo.NoCompression()
|
||||
}
|
||||
}
|
||||
176
ee/condensedmeter/kafkacondensedmeter/provider_test.go
Normal file
176
ee/condensedmeter/kafkacondensedmeter/provider_test.go
Normal file
@@ -0,0 +1,176 @@
|
||||
package kafkacondensedmeter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
cmock "github.com/SigNoz/clickhouse-go-mock"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/twmb/franz-go/pkg/kfake"
|
||||
"github.com/twmb/franz-go/pkg/kgo"
|
||||
"go.opentelemetry.io/collector/pdata/pmetric"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/condensedmeter"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
"github.com/SigNoz/signoz/pkg/types/condensedmetertypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/condensedmetertypes/condensedmetertypestest"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/SigNoz/signoz/pkg/version"
|
||||
)
|
||||
|
||||
// Polls the topic until it has wantRecords records or the timeout passes.
|
||||
func consumeMetrics(t *testing.T, brokers []string, topic string, wantRecords int, timeout time.Duration) []pmetric.Metrics {
|
||||
t.Helper()
|
||||
|
||||
client, err := kgo.NewClient(
|
||||
kgo.SeedBrokers(brokers...),
|
||||
kgo.ConsumeTopics(topic),
|
||||
kgo.ConsumeResetOffset(kgo.NewOffset().AtStart()),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
defer client.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
metrics := []pmetric.Metrics{}
|
||||
for len(metrics) < wantRecords && ctx.Err() == nil {
|
||||
for _, record := range client.PollFetches(ctx).Records() {
|
||||
recordMetrics, err := (&pmetric.ProtoUnmarshaler{}).UnmarshalMetrics(record.Value)
|
||||
require.NoError(t, err)
|
||||
metrics = append(metrics, recordMetrics)
|
||||
}
|
||||
}
|
||||
|
||||
return metrics
|
||||
}
|
||||
|
||||
func TestReportOrg(t *testing.T) {
|
||||
orgID := valuer.GenerateUUID()
|
||||
now := time.Date(2026, 11, 12, 13, 30, 0, 0, time.UTC)
|
||||
checkpointHour := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
|
||||
// 11:00 is closed at 13:30 (11:00 + 2h = 13:00); 12:00 is not (14:00)
|
||||
reportedHour := time.Date(2026, 11, 12, 11, 0, 0, 0, time.UTC)
|
||||
notFound := errors.New(errors.TypeNotFound, condensedmetertypes.ErrCodeCheckpointNotFound, "no checkpoint")
|
||||
condensedColumns := []cmock.ColumnType{{Name: "dimension_0", Type: "String"}, {Name: "dimension_1", Type: "String"}, {Name: "count", Type: "UInt64"}}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
checkpoint *condensedmetertypes.Checkpoint
|
||||
firstIngestedMs int64
|
||||
condensedRows [][]any
|
||||
condensedErr error
|
||||
expectedUpsert *time.Time
|
||||
expectedRecords int
|
||||
expectedDataPoint []int64
|
||||
expectedErr bool
|
||||
}{
|
||||
{
|
||||
name: "NoCheckpointNoIngestedRows_NothingReported",
|
||||
firstIngestedMs: 0,
|
||||
},
|
||||
{
|
||||
name: "Checkpoint_ClosedHourSentAndCheckpointAdvanced",
|
||||
checkpoint: condensedmetertypes.NewCheckpoint(orgID, checkpointHour),
|
||||
condensedRows: [][]any{{"frontend", "host-1", uint64(31)}, {"checkout", "", uint64(4)}},
|
||||
expectedUpsert: &reportedHour,
|
||||
expectedRecords: 1,
|
||||
expectedDataPoint: []int64{31, 4},
|
||||
},
|
||||
{
|
||||
name: "Checkpoint_LargeHourSplitAcrossRecords",
|
||||
checkpoint: condensedmetertypes.NewCheckpoint(orgID, checkpointHour),
|
||||
condensedRows: [][]any{{"frontend", "host-1", uint64(31)}, {"checkout", "", uint64(4)}, {"cart", "host-2", uint64(9)}},
|
||||
expectedUpsert: &reportedHour,
|
||||
expectedRecords: 2,
|
||||
expectedDataPoint: []int64{31, 4, 9},
|
||||
},
|
||||
{
|
||||
name: "Checkpoint_EmptyHourNotSentCheckpointAdvanced",
|
||||
checkpoint: condensedmetertypes.NewCheckpoint(orgID, checkpointHour),
|
||||
condensedRows: [][]any{},
|
||||
expectedUpsert: &reportedHour,
|
||||
},
|
||||
{
|
||||
name: "Checkpoint_QueryFails_CheckpointNotAdvanced",
|
||||
checkpoint: condensedmetertypes.NewCheckpoint(orgID, checkpointHour),
|
||||
condensedErr: errors.New(errors.TypeInternal, errors.CodeInternal, "clickhouse down"),
|
||||
expectedErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
cluster, err := kfake.NewCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, "tenant_meter"))
|
||||
require.NoError(t, err)
|
||||
defer cluster.Close()
|
||||
|
||||
telemetryStore := telemetrystoretest.New(telemetrystore.Config{}, sqlmock.QueryMatcherRegexp)
|
||||
store := condensedmetertypestest.NewMockStore(t)
|
||||
if testCase.checkpoint != nil {
|
||||
store.EXPECT().Get(mock.Anything, orgID).Return(testCase.checkpoint, nil)
|
||||
} else {
|
||||
store.EXPECT().Get(mock.Anything, orgID).Return(nil, notFound)
|
||||
telemetryStore.Mock().ExpectQueryRow(regexp.QuoteMeta("min(unix_milli)")).
|
||||
WillReturnRow(cmock.NewRow([]cmock.ColumnType{{Name: "min", Type: "Int64"}}, []any{testCase.firstIngestedMs}))
|
||||
}
|
||||
|
||||
if testCase.condensedRows != nil || testCase.condensedErr != nil {
|
||||
query := telemetryStore.Mock().ExpectQuery(regexp.QuoteMeta("distributed_time_series_v4_reduced")).
|
||||
WithArgs(reportedHour.UnixMilli(), reportedHour.Add(time.Hour).UnixMilli(), reportedHour.UnixMilli(), reportedHour.Add(time.Hour).UnixMilli(), reportedHour.UnixMilli(), reportedHour.Add(time.Hour).UnixMilli())
|
||||
if testCase.condensedErr != nil {
|
||||
query.WillReturnError(testCase.condensedErr)
|
||||
} else {
|
||||
query.WillReturnRows(cmock.NewRows(condensedColumns, testCase.condensedRows))
|
||||
}
|
||||
}
|
||||
|
||||
if testCase.expectedUpsert != nil {
|
||||
store.EXPECT().Upsert(mock.Anything, mock.MatchedBy(func(checkpoint *condensedmetertypes.Checkpoint) bool {
|
||||
return checkpoint.OrgID == orgID && checkpoint.LastReportedHour.Equal(*testCase.expectedUpsert)
|
||||
})).Return(nil)
|
||||
}
|
||||
|
||||
config := condensedmeter.Config{
|
||||
Provider: "kafka",
|
||||
Interval: time.Hour,
|
||||
Dimensions: []string{"service.name", "host.name"},
|
||||
BatchSize: 2,
|
||||
Kafka: condensedmeter.KafkaConfig{Brokers: cluster.ListenAddrs(), Topic: "tenant_meter", ClientID: "signoz-test", Compression: "none"},
|
||||
}
|
||||
provider, err := newProvider(context.Background(), instrumentationtest.New().ToProviderSettings(), config, nil, nil, nil, telemetryStore, store, version.Info)
|
||||
require.NoError(t, err)
|
||||
defer provider.client.Close()
|
||||
|
||||
err = provider.reportOrg(context.Background(), orgID, now)
|
||||
if testCase.expectedErr {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
assert.NoError(t, telemetryStore.Mock().ExpectationsWereMet())
|
||||
|
||||
metrics := consumeMetrics(t, cluster.ListenAddrs(), "tenant_meter", testCase.expectedRecords+1, 500*time.Millisecond)
|
||||
require.Len(t, metrics, testCase.expectedRecords)
|
||||
|
||||
dataPoints := []int64{}
|
||||
for _, recordMetrics := range metrics {
|
||||
points := recordMetrics.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints()
|
||||
for i := 0; i < points.Len(); i++ {
|
||||
dataPoints = append(dataPoints, points.At(i).IntValue())
|
||||
assert.Equal(t, now, points.At(i).Timestamp().AsTime())
|
||||
}
|
||||
}
|
||||
assert.ElementsMatch(t, testCase.expectedDataPoint, dataPoints)
|
||||
})
|
||||
}
|
||||
}
|
||||
43
ee/condensedmeter/kafkacondensedmeter/store.go
Normal file
43
ee/condensedmeter/kafkacondensedmeter/store.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package kafkacondensedmeter
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/types/condensedmetertypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
type store struct {
|
||||
sqlstore sqlstore.SQLStore
|
||||
}
|
||||
|
||||
func newStore(sqlstore sqlstore.SQLStore) condensedmetertypes.Store {
|
||||
return &store{sqlstore: sqlstore}
|
||||
}
|
||||
|
||||
func (store *store) Get(ctx context.Context, orgID valuer.UUID) (*condensedmetertypes.Checkpoint, error) {
|
||||
checkpoint := new(condensedmetertypes.Checkpoint)
|
||||
err := store.sqlstore.
|
||||
BunDBCtx(ctx).
|
||||
NewSelect().
|
||||
Model(checkpoint).
|
||||
Where("org_id = ?", orgID).
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
return nil, store.sqlstore.WrapNotFoundErrf(err, condensedmetertypes.ErrCodeCheckpointNotFound, "no condensed meter checkpoint found for org %q", orgID.StringValue())
|
||||
}
|
||||
return checkpoint, nil
|
||||
}
|
||||
|
||||
func (store *store) Upsert(ctx context.Context, checkpoint *condensedmetertypes.Checkpoint) error {
|
||||
_, err := store.sqlstore.
|
||||
BunDBCtx(ctx).
|
||||
NewInsert().
|
||||
Model(checkpoint).
|
||||
On("CONFLICT (org_id) DO UPDATE").
|
||||
Set("last_reported_hour = EXCLUDED.last_reported_hour").
|
||||
Set("updated_at = EXCLUDED.updated_at").
|
||||
Exec(ctx)
|
||||
return err
|
||||
}
|
||||
14
pkg/condensedmeter/condensedmeter.go
Normal file
14
pkg/condensedmeter/condensedmeter.go
Normal file
@@ -0,0 +1,14 @@
|
||||
package condensedmeter
|
||||
|
||||
import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrCodeInvalidInput = errors.MustNewCode("condensedmeter_invalid_input")
|
||||
)
|
||||
|
||||
type Reporter interface {
|
||||
factory.ServiceWithHealthy
|
||||
}
|
||||
85
pkg/condensedmeter/config.go
Normal file
85
pkg/condensedmeter/config.go
Normal file
@@ -0,0 +1,85 @@
|
||||
package condensedmeter
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
)
|
||||
|
||||
var _ factory.Config = (*Config)(nil)
|
||||
|
||||
type Config struct {
|
||||
Provider string `mapstructure:"provider"`
|
||||
|
||||
Interval time.Duration `mapstructure:"interval"`
|
||||
|
||||
// Dimensions are the resource attributes the meter connector records as meter labels.
|
||||
// They must match the connector's dimensions, so condensed rows slice like every other meter row.
|
||||
Dimensions []string `mapstructure:"dimensions"`
|
||||
|
||||
// BatchSize is the maximum number of data points per Kafka record.
|
||||
BatchSize int `mapstructure:"batch_size"`
|
||||
|
||||
Kafka KafkaConfig `mapstructure:"kafka"`
|
||||
}
|
||||
|
||||
type KafkaConfig struct {
|
||||
// Brokers are the seed broker addresses (host:port) used to discover the cluster.
|
||||
Brokers []string `mapstructure:"brokers"`
|
||||
|
||||
// Topic must already exist; the producer does not create it.
|
||||
Topic string `mapstructure:"topic"`
|
||||
|
||||
// ClientID identifies this producer in broker logs and metrics.
|
||||
ClientID string `mapstructure:"client_id"`
|
||||
|
||||
// Compression is the codec for produced batches: none, gzip, snappy, lz4 or zstd.
|
||||
Compression string `mapstructure:"compression"`
|
||||
}
|
||||
|
||||
func newConfig() factory.Config {
|
||||
return Config{
|
||||
Provider: "noop",
|
||||
Interval: time.Hour,
|
||||
Dimensions: []string{"service.name", "deployment.environment", "host.name", "signoz.workspace.key.id"},
|
||||
BatchSize: 100,
|
||||
Kafka: KafkaConfig{
|
||||
Brokers: []string{"localhost:9092"},
|
||||
ClientID: "signoz",
|
||||
Compression: "lz4",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func NewConfigFactory() factory.ConfigFactory {
|
||||
return factory.NewConfigFactory(factory.MustNewName("condensedmeter"), newConfig)
|
||||
}
|
||||
|
||||
func (c Config) Validate() error {
|
||||
if c.Interval < 10*time.Minute || c.Interval > time.Hour {
|
||||
return errors.New(errors.TypeInvalidInput, ErrCodeInvalidInput, "condensedmeter::interval must be between 10m and 1h")
|
||||
}
|
||||
|
||||
if c.BatchSize <= 0 {
|
||||
return errors.New(errors.TypeInvalidInput, ErrCodeInvalidInput, "condensedmeter::batch_size must be greater than 0")
|
||||
}
|
||||
|
||||
if c.Provider == "kafka" {
|
||||
if len(c.Kafka.Brokers) == 0 {
|
||||
return errors.New(errors.TypeInvalidInput, ErrCodeInvalidInput, "condensedmeter::kafka::brokers must be set when provider is kafka")
|
||||
}
|
||||
|
||||
if c.Kafka.Topic == "" {
|
||||
return errors.New(errors.TypeInvalidInput, ErrCodeInvalidInput, "condensedmeter::kafka::topic must be set when provider is kafka")
|
||||
}
|
||||
|
||||
switch c.Kafka.Compression {
|
||||
case "none", "gzip", "snappy", "lz4", "zstd":
|
||||
default:
|
||||
return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidInput, "condensedmeter::kafka::compression must be one of none, gzip, snappy, lz4 or zstd, got %q", c.Kafka.Compression)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
39
pkg/condensedmeter/noopcondensedmeter/provider.go
Normal file
39
pkg/condensedmeter/noopcondensedmeter/provider.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package noopcondensedmeter
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/condensedmeter"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
)
|
||||
|
||||
type provider struct {
|
||||
healthyC chan struct{}
|
||||
stopC chan struct{}
|
||||
}
|
||||
|
||||
func NewFactory() factory.ProviderFactory[condensedmeter.Reporter, condensedmeter.Config] {
|
||||
return factory.NewProviderFactory(factory.MustNewName("noop"), New)
|
||||
}
|
||||
|
||||
func New(_ context.Context, _ factory.ProviderSettings, _ condensedmeter.Config) (condensedmeter.Reporter, error) {
|
||||
return &provider{
|
||||
healthyC: make(chan struct{}),
|
||||
stopC: make(chan struct{}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (provider *provider) Start(_ context.Context) error {
|
||||
close(provider.healthyC)
|
||||
<-provider.stopC
|
||||
return nil
|
||||
}
|
||||
|
||||
func (provider *provider) Stop(_ context.Context) error {
|
||||
close(provider.stopC)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (provider *provider) Healthy() <-chan struct{} {
|
||||
return provider.healthyC
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/auditor"
|
||||
"github.com/SigNoz/signoz/pkg/authz"
|
||||
"github.com/SigNoz/signoz/pkg/cache"
|
||||
"github.com/SigNoz/signoz/pkg/condensedmeter"
|
||||
"github.com/SigNoz/signoz/pkg/config"
|
||||
"github.com/SigNoz/signoz/pkg/emailing"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
@@ -139,6 +140,9 @@ type Config struct {
|
||||
// MeterReporter config
|
||||
MeterReporter meterreporter.Config `mapstructure:"meterreporter"`
|
||||
|
||||
// CondensedMeter config
|
||||
CondensedMeter condensedmeter.Config `mapstructure:"condensedmeter"`
|
||||
|
||||
// CloudIntegration config
|
||||
CloudIntegration cloudintegration.Config `mapstructure:"cloudintegration"`
|
||||
|
||||
@@ -180,6 +184,7 @@ func NewConfig(ctx context.Context, logger *slog.Logger, resolverConfig config.R
|
||||
serviceaccount.NewConfigFactory(),
|
||||
auditor.NewConfigFactory(),
|
||||
meterreporter.NewConfigFactory(),
|
||||
condensedmeter.NewConfigFactory(),
|
||||
cloudintegration.NewConfigFactory(),
|
||||
tracedetail.NewConfigFactory(),
|
||||
authz.NewConfigFactory(),
|
||||
|
||||
@@ -16,6 +16,8 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/cache"
|
||||
"github.com/SigNoz/signoz/pkg/cache/memorycache"
|
||||
"github.com/SigNoz/signoz/pkg/cache/rediscache"
|
||||
"github.com/SigNoz/signoz/pkg/condensedmeter"
|
||||
"github.com/SigNoz/signoz/pkg/condensedmeter/noopcondensedmeter"
|
||||
"github.com/SigNoz/signoz/pkg/emailing"
|
||||
"github.com/SigNoz/signoz/pkg/emailing/noopemailing"
|
||||
"github.com/SigNoz/signoz/pkg/emailing/smtpemailing"
|
||||
@@ -275,6 +277,7 @@ func NewSQLMigrationProviderFactories(
|
||||
sqlmigration.NewMigrateRulesV1ToV2Alpha1Factory(sqlstore),
|
||||
sqlmigration.NewAddLLMPricingRuleTuplesFactory(sqlstore),
|
||||
sqlmigration.NewRemoveUnsupportedQuickFiltersFactory(),
|
||||
sqlmigration.NewAddCondensedMeterCheckpointFactory(sqlstore, sqlschema),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -429,6 +432,12 @@ func NewMeterReporterProviderFactories() factory.NamedMap[factory.ProviderFactor
|
||||
)
|
||||
}
|
||||
|
||||
func NewCondensedMeterProviderFactories() factory.NamedMap[factory.ProviderFactory[condensedmeter.Reporter, condensedmeter.Config]] {
|
||||
return factory.MustNewNamedMap(
|
||||
noopcondensedmeter.NewFactory(),
|
||||
)
|
||||
}
|
||||
|
||||
func NewFlaggerProviderFactories(registry featuretypes.Registry) factory.NamedMap[factory.ProviderFactory[flagger.FlaggerProvider, flagger.Config]] {
|
||||
return factory.MustNewNamedMap(
|
||||
configflagger.NewFactory(registry),
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/authn/authnstore/sqlauthnstore"
|
||||
"github.com/SigNoz/signoz/pkg/authz"
|
||||
"github.com/SigNoz/signoz/pkg/cache"
|
||||
"github.com/SigNoz/signoz/pkg/condensedmeter"
|
||||
"github.com/SigNoz/signoz/pkg/emailing"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
@@ -99,6 +100,7 @@ type SigNoz struct {
|
||||
Gateway gateway.Gateway
|
||||
Auditor auditor.Auditor
|
||||
MeterReporter meterreporter.Reporter
|
||||
CondensedMeter condensedmeter.Reporter
|
||||
}
|
||||
|
||||
// newQueryStack assembles the query stack once and returns, in order: the shared
|
||||
@@ -184,6 +186,7 @@ func New(
|
||||
gatewayProviderFactory func(licensing.Licensing) factory.ProviderFactory[gateway.Gateway, gateway.Config],
|
||||
auditorProviderFactories func(licensing.Licensing) factory.NamedMap[factory.ProviderFactory[auditor.Auditor, auditor.Config]],
|
||||
meterReporterProviderFactories func(context.Context, factory.ProviderSettings, flagger.Flagger, licensing.Licensing, telemetrystore.TelemetryStore, retention.Getter, organization.Getter, zeus.Zeus) (factory.NamedMap[factory.ProviderFactory[meterreporter.Reporter, meterreporter.Config]], string),
|
||||
condensedMeterProviderFactories func(flagger.Flagger, licensing.Licensing, organization.Getter, telemetrystore.TelemetryStore, sqlstore.SQLStore) factory.NamedMap[factory.ProviderFactory[condensedmeter.Reporter, condensedmeter.Config]],
|
||||
querierHandlerCallback func(factory.ProviderSettings, querier.Querier, analytics.Analytics) querier.Handler,
|
||||
cloudIntegrationCallback func(sqlstore.SQLStore, dashboard.Module, global.Global, zeus.Zeus, gateway.Gateway, licensing.Licensing, serviceaccount.Module, cloudintegration.Config) (cloudintegration.Module, error),
|
||||
metricReductionRuleModuleCallback func(sqlstore.SQLStore, telemetrystore.TelemetryStore, dashboard.Module, queryparser.QueryParser, licensing.Licensing, flagger.Flagger, telemetrytypes.MetadataStore, factory.ProviderSettings, int) metricreductionrule.Module,
|
||||
@@ -501,6 +504,12 @@ func New(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Initialize condensed meter from the variant-specific provider factories
|
||||
condensedMeter, err := factory.NewProviderFromNamedMap(ctx, providerSettings, config.CondensedMeter, condensedMeterProviderFactories(flagger, licensing, orgGetter, telemetrystore, sqlstore), config.CondensedMeter.Provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Initialize authns
|
||||
store := sqlauthnstore.NewStore(sqlstore)
|
||||
authNs, err := authNsCallback(ctx, providerSettings, store, licensing)
|
||||
@@ -606,6 +615,7 @@ func New(
|
||||
factory.NewNamedService(factory.MustNewName("user"), userService, factory.MustNewName("authz")),
|
||||
factory.NewNamedService(factory.MustNewName("auditor"), auditor),
|
||||
factory.NewNamedService(factory.MustNewName("meterreporter"), meterReporter, factory.MustNewName("licensing")),
|
||||
factory.NewNamedService(factory.MustNewName("condensedmeter"), condensedMeter, factory.MustNewName("licensing")),
|
||||
factory.NewNamedService(factory.MustNewName("ruler"), rulerInstance),
|
||||
factory.NewNamedService(factory.MustNewName("systemdashboard"), impldashboard.NewService(providerSettings, dashboard, orgGetter)),
|
||||
factory.NewNamedService(factory.MustNewName("spanmappergroup"), implspanmapper.NewService(providerSettings, spanMapperModule, orgGetter)),
|
||||
@@ -667,5 +677,6 @@ func New(
|
||||
Gateway: gateway,
|
||||
Auditor: auditor,
|
||||
MeterReporter: meterReporter,
|
||||
CondensedMeter: condensedMeter,
|
||||
}, nil
|
||||
}
|
||||
|
||||
78
pkg/sqlmigration/148_add_condensed_meter_checkpoint.go
Normal file
78
pkg/sqlmigration/148_add_condensed_meter_checkpoint.go
Normal file
@@ -0,0 +1,78 @@
|
||||
package sqlmigration
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
"github.com/uptrace/bun/migrate"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/sqlschema"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
)
|
||||
|
||||
type addCondensedMeterCheckpoint struct {
|
||||
sqlschema sqlschema.SQLSchema
|
||||
sqlstore sqlstore.SQLStore
|
||||
}
|
||||
|
||||
func NewAddCondensedMeterCheckpointFactory(sqlstore sqlstore.SQLStore, sqlschema sqlschema.SQLSchema) factory.ProviderFactory[SQLMigration, Config] {
|
||||
return factory.NewProviderFactory(factory.MustNewName("add_condensed_meter_checkpoint"), func(_ context.Context, _ factory.ProviderSettings, _ Config) (SQLMigration, error) {
|
||||
return &addCondensedMeterCheckpoint{
|
||||
sqlschema: sqlschema,
|
||||
sqlstore: sqlstore,
|
||||
}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (migration *addCondensedMeterCheckpoint) Register(migrations *migrate.Migrations) error {
|
||||
if err := migrations.Register(migration.Up, migration.Down); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (migration *addCondensedMeterCheckpoint) Up(ctx context.Context, db *bun.DB) error {
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
|
||||
sqls := [][]byte{}
|
||||
|
||||
tableSQLs := migration.sqlschema.Operator().CreateTable(&sqlschema.Table{
|
||||
Name: "condensed_meter_checkpoint",
|
||||
Columns: []*sqlschema.Column{
|
||||
{Name: "org_id", DataType: sqlschema.DataTypeText, Nullable: false},
|
||||
{Name: "last_reported_hour", DataType: sqlschema.DataTypeTimestamp, Nullable: false},
|
||||
{Name: "updated_at", DataType: sqlschema.DataTypeTimestamp, Nullable: false},
|
||||
},
|
||||
PrimaryKeyConstraint: &sqlschema.PrimaryKeyConstraint{
|
||||
ColumnNames: []sqlschema.ColumnName{"org_id"},
|
||||
},
|
||||
ForeignKeyConstraints: []*sqlschema.ForeignKeyConstraint{
|
||||
{
|
||||
ReferencingColumnName: sqlschema.ColumnName("org_id"),
|
||||
ReferencedTableName: sqlschema.TableName("organizations"),
|
||||
ReferencedColumnName: sqlschema.ColumnName("id"),
|
||||
},
|
||||
},
|
||||
})
|
||||
sqls = append(sqls, tableSQLs...)
|
||||
|
||||
for _, sql := range sqls {
|
||||
if _, err := tx.ExecContext(ctx, string(sql)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (migration *addCondensedMeterCheckpoint) Down(context.Context, *bun.DB) error {
|
||||
return nil
|
||||
}
|
||||
36
pkg/types/condensedmetertypes/checkpoint.go
Normal file
36
pkg/types/condensedmetertypes/checkpoint.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package condensedmetertypes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/uptrace/bun"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrCodeCheckpointNotFound = errors.MustNewCode("condensed_meter_checkpoint_not_found")
|
||||
)
|
||||
|
||||
// Checkpoint is the last hour whose condensed rows were sent for an org.
|
||||
type Checkpoint struct {
|
||||
bun.BaseModel `bun:"table:condensed_meter_checkpoint"`
|
||||
|
||||
OrgID valuer.UUID `bun:"org_id,pk,type:text"`
|
||||
LastReportedHour time.Time `bun:"last_reported_hour"`
|
||||
UpdatedAt time.Time `bun:"updated_at"`
|
||||
}
|
||||
|
||||
type Store interface {
|
||||
Get(ctx context.Context, orgID valuer.UUID) (*Checkpoint, error)
|
||||
Upsert(ctx context.Context, checkpoint *Checkpoint) error
|
||||
}
|
||||
|
||||
func NewCheckpoint(orgID valuer.UUID, hour time.Time) *Checkpoint {
|
||||
return &Checkpoint{
|
||||
OrgID: orgID,
|
||||
LastReportedHour: hour,
|
||||
UpdatedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
165
pkg/types/condensedmetertypes/condensedmetertypestest/store.go
Normal file
165
pkg/types/condensedmetertypes/condensedmetertypestest/store.go
Normal file
@@ -0,0 +1,165 @@
|
||||
// Code generated by mockery; DO NOT EDIT.
|
||||
// github.com/vektra/mockery
|
||||
// template: testify
|
||||
|
||||
package condensedmetertypestest
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/condensedmetertypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// NewMockStore creates a new instance of MockStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewMockStore(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *MockStore {
|
||||
mock := &MockStore{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
|
||||
// MockStore is an autogenerated mock type for the Store type
|
||||
type MockStore struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
type MockStore_Expecter struct {
|
||||
mock *mock.Mock
|
||||
}
|
||||
|
||||
func (_m *MockStore) EXPECT() *MockStore_Expecter {
|
||||
return &MockStore_Expecter{mock: &_m.Mock}
|
||||
}
|
||||
|
||||
// Get provides a mock function for the type MockStore
|
||||
func (_mock *MockStore) Get(ctx context.Context, orgID valuer.UUID) (*condensedmetertypes.Checkpoint, error) {
|
||||
ret := _mock.Called(ctx, orgID)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Get")
|
||||
}
|
||||
|
||||
var r0 *condensedmetertypes.Checkpoint
|
||||
var r1 error
|
||||
if returnFunc, ok := ret.Get(0).(func(context.Context, valuer.UUID) (*condensedmetertypes.Checkpoint, error)); ok {
|
||||
return returnFunc(ctx, orgID)
|
||||
}
|
||||
if returnFunc, ok := ret.Get(0).(func(context.Context, valuer.UUID) *condensedmetertypes.Checkpoint); ok {
|
||||
r0 = returnFunc(ctx, orgID)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*condensedmetertypes.Checkpoint)
|
||||
}
|
||||
}
|
||||
if returnFunc, ok := ret.Get(1).(func(context.Context, valuer.UUID) error); ok {
|
||||
r1 = returnFunc(ctx, orgID)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockStore_Get_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Get'
|
||||
type MockStore_Get_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Get is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - orgID valuer.UUID
|
||||
func (_e *MockStore_Expecter) Get(ctx interface{}, orgID interface{}) *MockStore_Get_Call {
|
||||
return &MockStore_Get_Call{Call: _e.mock.On("Get", ctx, orgID)}
|
||||
}
|
||||
|
||||
func (_c *MockStore_Get_Call) Run(run func(ctx context.Context, orgID valuer.UUID)) *MockStore_Get_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 context.Context
|
||||
if args[0] != nil {
|
||||
arg0 = args[0].(context.Context)
|
||||
}
|
||||
var arg1 valuer.UUID
|
||||
if args[1] != nil {
|
||||
arg1 = args[1].(valuer.UUID)
|
||||
}
|
||||
run(
|
||||
arg0,
|
||||
arg1,
|
||||
)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockStore_Get_Call) Return(checkpoint *condensedmetertypes.Checkpoint, err error) *MockStore_Get_Call {
|
||||
_c.Call.Return(checkpoint, err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockStore_Get_Call) RunAndReturn(run func(ctx context.Context, orgID valuer.UUID) (*condensedmetertypes.Checkpoint, error)) *MockStore_Get_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Upsert provides a mock function for the type MockStore
|
||||
func (_mock *MockStore) Upsert(ctx context.Context, checkpoint *condensedmetertypes.Checkpoint) error {
|
||||
ret := _mock.Called(ctx, checkpoint)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Upsert")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if returnFunc, ok := ret.Get(0).(func(context.Context, *condensedmetertypes.Checkpoint) error); ok {
|
||||
r0 = returnFunc(ctx, checkpoint)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockStore_Upsert_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Upsert'
|
||||
type MockStore_Upsert_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Upsert is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - checkpoint *condensedmetertypes.Checkpoint
|
||||
func (_e *MockStore_Expecter) Upsert(ctx interface{}, checkpoint interface{}) *MockStore_Upsert_Call {
|
||||
return &MockStore_Upsert_Call{Call: _e.mock.On("Upsert", ctx, checkpoint)}
|
||||
}
|
||||
|
||||
func (_c *MockStore_Upsert_Call) Run(run func(ctx context.Context, checkpoint *condensedmetertypes.Checkpoint)) *MockStore_Upsert_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 context.Context
|
||||
if args[0] != nil {
|
||||
arg0 = args[0].(context.Context)
|
||||
}
|
||||
var arg1 *condensedmetertypes.Checkpoint
|
||||
if args[1] != nil {
|
||||
arg1 = args[1].(*condensedmetertypes.Checkpoint)
|
||||
}
|
||||
run(
|
||||
arg0,
|
||||
arg1,
|
||||
)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockStore_Upsert_Call) Return(err error) *MockStore_Upsert_Call {
|
||||
_c.Call.Return(err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockStore_Upsert_Call) RunAndReturn(run func(ctx context.Context, checkpoint *condensedmetertypes.Checkpoint) error) *MockStore_Upsert_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
22
pkg/types/condensedmetertypes/hour.go
Normal file
22
pkg/types/condensedmetertypes/hour.go
Normal file
@@ -0,0 +1,22 @@
|
||||
package condensedmetertypes
|
||||
|
||||
import "time"
|
||||
|
||||
// CondensingDelay is how long after an hour starts its condensed rows are final.
|
||||
// Condensing of a minute finishes about 10 minutes after it, so the last minute of an hour is final within 2h of its start.
|
||||
const CondensingDelay = 2 * time.Hour
|
||||
|
||||
// FirstHour is the hour after the first ingested meter row. Before it, datapoint.count meant ingested, so adding condensed rows would count them twice.
|
||||
func FirstHour(firstIngested time.Time) time.Time {
|
||||
return firstIngested.UTC().Truncate(time.Hour).Add(time.Hour)
|
||||
}
|
||||
|
||||
// ClosedHours returns the hours from next whose condensed rows are final at now, oldest first.
|
||||
func ClosedHours(next time.Time, now time.Time) []time.Time {
|
||||
hours := []time.Time{}
|
||||
for hour := next; !hour.Add(CondensingDelay).After(now); hour = hour.Add(time.Hour) {
|
||||
hours = append(hours, hour)
|
||||
}
|
||||
|
||||
return hours
|
||||
}
|
||||
48
pkg/types/condensedmetertypes/hour_test.go
Normal file
48
pkg/types/condensedmetertypes/hour_test.go
Normal file
@@ -0,0 +1,48 @@
|
||||
package condensedmetertypes
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestFirstHour(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
firstIngested time.Time
|
||||
expected time.Time
|
||||
}{
|
||||
{name: "MidHour_NextHour", firstIngested: time.Date(2026, 11, 12, 9, 39, 38, 0, time.UTC), expected: time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)},
|
||||
{name: "OnTheHour_NextHour", firstIngested: time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC), expected: time.Date(2026, 11, 12, 11, 0, 0, 0, time.UTC)},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
assert.Equal(t, testCase.expected, FirstHour(testCase.firstIngested))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClosedHours(t *testing.T) {
|
||||
next := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
now time.Time
|
||||
expected []time.Time
|
||||
}{
|
||||
// 10:00 + 2h = 12:00 is after 11:59
|
||||
{name: "NotClosed_None", now: time.Date(2026, 11, 12, 11, 59, 0, 0, time.UTC), expected: []time.Time{}},
|
||||
// 10:00 + 2h = 12:00
|
||||
{name: "ClosedAtDelay_One", now: time.Date(2026, 11, 12, 12, 0, 0, 0, time.UTC), expected: []time.Time{next}},
|
||||
// 12:00 + 2h = 14:00 is before 14:30, 13:00 + 2h = 15:00 is not
|
||||
{name: "Backlog_OldestFirst", now: time.Date(2026, 11, 12, 14, 30, 0, 0, time.UTC), expected: []time.Time{next, next.Add(time.Hour), next.Add(2 * time.Hour)}},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
assert.Equal(t, testCase.expected, ClosedHours(next, testCase.now))
|
||||
})
|
||||
}
|
||||
}
|
||||
49
pkg/types/condensedmetertypes/metrics.go
Normal file
49
pkg/types/condensedmetertypes/metrics.go
Normal file
@@ -0,0 +1,49 @@
|
||||
package condensedmetertypes
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/collector/pdata/pcommon"
|
||||
"go.opentelemetry.io/collector/pdata/pmetric"
|
||||
)
|
||||
|
||||
const (
|
||||
metricNameMetricsDataPointsCount = "signoz.meter.metric.datapoint.count"
|
||||
metricDescMetricsDataPointsCount = "The number of data points indexed."
|
||||
)
|
||||
|
||||
// Row is the number of condensed rows of one dimension set in one hour.
|
||||
// Values are positional, in the order of the configured dimensions; an empty value means the dimension is absent.
|
||||
type Row struct {
|
||||
Values []string
|
||||
Count uint64
|
||||
}
|
||||
|
||||
// NewPMetrics shapes rows like the meter connector's output (dimensions as data point attributes, delta monotonic sum), so they land in the same meter table and slice the same way.
|
||||
func NewPMetrics(rows []Row, dimensions []string, timestamp time.Time, scopeName string, scopeVersion string) pmetric.Metrics {
|
||||
metrics := pmetric.NewMetrics()
|
||||
scopeMetrics := metrics.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty()
|
||||
scopeMetrics.Scope().SetName(scopeName)
|
||||
scopeMetrics.Scope().SetVersion(scopeVersion)
|
||||
|
||||
metric := scopeMetrics.Metrics().AppendEmpty()
|
||||
metric.SetName(metricNameMetricsDataPointsCount)
|
||||
metric.SetDescription(metricDescMetricsDataPointsCount)
|
||||
metric.SetUnit("1")
|
||||
sum := metric.SetEmptySum()
|
||||
sum.SetAggregationTemporality(pmetric.AggregationTemporalityDelta)
|
||||
sum.SetIsMonotonic(true)
|
||||
|
||||
for _, row := range rows {
|
||||
dataPoint := sum.DataPoints().AppendEmpty()
|
||||
dataPoint.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
|
||||
dataPoint.SetIntValue(int64(row.Count))
|
||||
for i, dimension := range dimensions {
|
||||
if row.Values[i] != "" {
|
||||
dataPoint.Attributes().PutStr(dimension, row.Values[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return metrics
|
||||
}
|
||||
57
pkg/types/condensedmetertypes/metrics_test.go
Normal file
57
pkg/types/condensedmetertypes/metrics_test.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package condensedmetertypes
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.opentelemetry.io/collector/pdata/pmetric"
|
||||
)
|
||||
|
||||
func TestNewPMetrics(t *testing.T) {
|
||||
timestamp := time.Date(2026, 11, 12, 13, 5, 0, 0, time.UTC)
|
||||
dimensions := []string{"service.name", "host.name"}
|
||||
rows := []Row{
|
||||
{Values: []string{"frontend", "host-1"}, Count: 31},
|
||||
{Values: []string{"checkout", ""}, Count: 4},
|
||||
}
|
||||
|
||||
metrics := NewPMetrics(rows, dimensions, timestamp, "signoz", "v0.0.1")
|
||||
|
||||
require.Equal(t, 1, metrics.ResourceMetrics().Len())
|
||||
resourceMetrics := metrics.ResourceMetrics().At(0)
|
||||
assert.Equal(t, 0, resourceMetrics.Resource().Attributes().Len())
|
||||
require.Equal(t, 1, resourceMetrics.ScopeMetrics().Len())
|
||||
scopeMetrics := resourceMetrics.ScopeMetrics().At(0)
|
||||
assert.Equal(t, "signoz", scopeMetrics.Scope().Name())
|
||||
assert.Equal(t, "v0.0.1", scopeMetrics.Scope().Version())
|
||||
require.Equal(t, 1, scopeMetrics.Metrics().Len())
|
||||
metric := scopeMetrics.Metrics().At(0)
|
||||
assert.Equal(t, "signoz.meter.metric.datapoint.count", metric.Name())
|
||||
require.Equal(t, pmetric.MetricTypeSum, metric.Type())
|
||||
assert.Equal(t, pmetric.AggregationTemporalityDelta, metric.Sum().AggregationTemporality())
|
||||
assert.True(t, metric.Sum().IsMonotonic())
|
||||
|
||||
dataPoints := metric.Sum().DataPoints()
|
||||
require.Equal(t, 2, dataPoints.Len())
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
index int
|
||||
value int64
|
||||
attributes map[string]any
|
||||
}{
|
||||
{name: "AllDimensions", index: 0, value: 31, attributes: map[string]any{"service.name": "frontend", "host.name": "host-1"}},
|
||||
{name: "EmptyDimension_Omitted", index: 1, value: 4, attributes: map[string]any{"service.name": "checkout"}},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
dataPoint := dataPoints.At(testCase.index)
|
||||
assert.Equal(t, testCase.value, dataPoint.IntValue())
|
||||
assert.Equal(t, timestamp, dataPoint.Timestamp().AsTime())
|
||||
assert.Equal(t, testCase.attributes, dataPoint.Attributes().AsRaw())
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user