Compare commits

..

1 Commits

27 changed files with 715 additions and 764 deletions

View File

@@ -2437,6 +2437,17 @@ components:
url:
type: string
type: object
DashboardTextVariableSpec:
properties:
constant:
type: boolean
display:
$ref: '#/components/schemas/VariableDisplay'
name:
type: string
value:
type: string
type: object
DashboardtypesAxes:
properties:
isLogScale:
@@ -2801,15 +2812,9 @@ components:
type: string
nullable: true
type: object
mode:
$ref: '#/components/schemas/DashboardtypesLegendMode'
position:
$ref: '#/components/schemas/DashboardtypesLegendPosition'
type: object
DashboardtypesLegendMode:
enum:
- list
type: string
DashboardtypesLegendPosition:
enum:
- bottom
@@ -2860,25 +2865,15 @@ components:
display:
$ref: '#/components/schemas/DashboardtypesDisplay'
name:
minLength: 1
type: string
plugin:
$ref: '#/components/schemas/DashboardtypesVariablePlugin'
sort:
$ref: '#/components/schemas/DashboardtypesListVariableSpecSort'
nullable: true
type: string
required:
- name
- display
type: object
DashboardtypesListVariableSpecSort:
enum:
- none
- alphabetical-asc
- alphabetical-desc
- numerical-asc
- numerical-desc
- alphabetical-ci-asc
- alphabetical-ci-desc
type: string
DashboardtypesListableDashboardForUserV2:
properties:
dashboards:
@@ -3388,13 +3383,8 @@ components:
DashboardtypesSpanGaps:
properties:
fillLessThan:
description: The maximum gap size to connect when fillOnlyBelow is true.
Gaps larger than this duration are left disconnected.
type: string
fillOnlyBelow:
description: Controls whether lines connect across null values. When false
(default), all gaps are connected. When true, only gaps smaller than fillLessThan
are connected.
type: boolean
type: object
DashboardtypesStorableDashboardData:
@@ -3442,20 +3432,6 @@ components:
- color
- columnName
type: object
DashboardtypesTextVariableSpec:
properties:
constant:
type: boolean
display:
$ref: '#/components/schemas/DashboardtypesDisplay'
name:
minLength: 1
type: string
value:
type: string
required:
- name
type: object
DashboardtypesThresholdFormat:
enum:
- text
@@ -3475,6 +3451,7 @@ components:
required:
- value
- color
- label
type: object
DashboardtypesTimePreference:
enum:
@@ -3559,11 +3536,23 @@ components:
discriminator:
mapping:
ListVariable: '#/components/schemas/DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDashboardtypesListVariableSpec'
TextVariable: '#/components/schemas/DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDashboardtypesTextVariableSpec'
TextVariable: '#/components/schemas/DashboardtypesVariableEnvelopeGithubComPersesSpecGoDashboardTextVariableSpec'
propertyName: kind
oneOf:
- $ref: '#/components/schemas/DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDashboardtypesListVariableSpec'
- $ref: '#/components/schemas/DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDashboardtypesTextVariableSpec'
- $ref: '#/components/schemas/DashboardtypesVariableEnvelopeGithubComPersesSpecGoDashboardTextVariableSpec'
type: object
DashboardtypesVariableEnvelopeGithubComPersesSpecGoDashboardTextVariableSpec:
properties:
kind:
enum:
- TextVariable
type: string
spec:
$ref: '#/components/schemas/DashboardTextVariableSpec'
required:
- kind
- spec
type: object
DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDashboardtypesListVariableSpec:
properties:
@@ -3577,18 +3566,6 @@ components:
- kind
- spec
type: object
DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDashboardtypesTextVariableSpec:
properties:
kind:
enum:
- TextVariable
type: string
spec:
$ref: '#/components/schemas/DashboardtypesTextVariableSpec'
required:
- kind
- spec
type: object
DashboardtypesVariablePlugin:
discriminator:
mapping:
@@ -7674,6 +7651,15 @@ components:
type: object
VariableDefaultValue:
type: object
VariableDisplay:
properties:
description:
type: string
hidden:
type: boolean
name:
type: string
type: object
ZeustypesGettableHost:
properties:
hosts:

View File

@@ -3154,6 +3154,37 @@ export interface DashboardLinkDTO {
url?: string;
}
export interface VariableDisplayDTO {
/**
* @type string
*/
description?: string;
/**
* @type boolean
*/
hidden?: boolean;
/**
* @type string
*/
name?: string;
}
export interface DashboardTextVariableSpecDTO {
/**
* @type boolean
*/
constant?: boolean;
display?: VariableDisplayDTO;
/**
* @type string
*/
name?: string;
/**
* @type string
*/
value?: string;
}
export interface DashboardtypesAxesDTO {
/**
* @type boolean
@@ -3185,9 +3216,6 @@ export interface DashboardtypesPanelFormattingDTO {
unit?: string;
}
export enum DashboardtypesLegendModeDTO {
list = 'list',
}
export enum DashboardtypesLegendPositionDTO {
bottom = 'bottom',
right = 'right',
@@ -3207,7 +3235,6 @@ export interface DashboardtypesLegendDTO {
* @type object,null
*/
customColors?: DashboardtypesLegendDTOCustomColors;
mode?: DashboardtypesLegendModeDTO;
position?: DashboardtypesLegendPositionDTO;
}
@@ -3219,7 +3246,7 @@ export interface DashboardtypesThresholdWithLabelDTO {
/**
* @type string
*/
label?: string;
label: string;
/**
* @type string
*/
@@ -3884,12 +3911,10 @@ export enum DashboardtypesLineStyleDTO {
export interface DashboardtypesSpanGapsDTO {
/**
* @type string
* @description The maximum gap size to connect when fillOnlyBelow is true. Gaps larger than this duration are left disconnected.
*/
fillLessThan?: string;
/**
* @type boolean
* @description Controls whether lines connect across null values. When false (default), all gaps are connected. When true, only gaps smaller than fillLessThan are connected.
*/
fillOnlyBelow?: boolean;
}
@@ -4521,15 +4546,6 @@ export type DashboardtypesVariablePluginDTO =
| DashboardtypesVariablePluginVariantGithubComSigNozSignozPkgTypesDashboardtypesQueryVariableSpecDTO
| DashboardtypesVariablePluginVariantGithubComSigNozSignozPkgTypesDashboardtypesCustomVariableSpecDTO;
export enum DashboardtypesListVariableSpecSortDTO {
none = 'none',
'alphabetical-asc' = 'alphabetical-asc',
'alphabetical-desc' = 'alphabetical-desc',
'numerical-asc' = 'numerical-asc',
'numerical-desc' = 'numerical-desc',
'alphabetical-ci-asc' = 'alphabetical-ci-asc',
'alphabetical-ci-desc' = 'alphabetical-ci-desc',
}
export interface DashboardtypesListVariableSpecDTO {
/**
* @type boolean
@@ -4548,14 +4564,16 @@ export interface DashboardtypesListVariableSpecDTO {
*/
customAllValue?: string;
defaultValue?: VariableDefaultValueDTO;
display?: DashboardtypesDisplayDTO;
display: DashboardtypesDisplayDTO;
/**
* @type string
* @minLength 1
*/
name: string;
name?: string;
plugin?: DashboardtypesVariablePluginDTO;
sort?: DashboardtypesListVariableSpecSortDTO;
/**
* @type string,null
*/
sort?: string | null;
}
export interface DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDashboardtypesListVariableSpecDTO {
@@ -4567,38 +4585,21 @@ export interface DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDash
spec: DashboardtypesListVariableSpecDTO;
}
export enum DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDashboardtypesTextVariableSpecDTOKind {
export enum DashboardtypesVariableEnvelopeGithubComPersesSpecGoDashboardTextVariableSpecDTOKind {
TextVariable = 'TextVariable',
}
export interface DashboardtypesTextVariableSpecDTO {
/**
* @type boolean
*/
constant?: boolean;
display?: DashboardtypesDisplayDTO;
/**
* @type string
* @minLength 1
*/
name: string;
/**
* @type string
*/
value?: string;
}
export interface DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDashboardtypesTextVariableSpecDTO {
export interface DashboardtypesVariableEnvelopeGithubComPersesSpecGoDashboardTextVariableSpecDTO {
/**
* @enum TextVariable
* @type string
*/
kind: DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDashboardtypesTextVariableSpecDTOKind;
spec: DashboardtypesTextVariableSpecDTO;
kind: DashboardtypesVariableEnvelopeGithubComPersesSpecGoDashboardTextVariableSpecDTOKind;
spec: DashboardTextVariableSpecDTO;
}
export type DashboardtypesVariableDTO =
| DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDashboardtypesListVariableSpecDTO
| DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDashboardtypesTextVariableSpecDTO;
| DashboardtypesVariableEnvelopeGithubComPersesSpecGoDashboardTextVariableSpecDTO;
export interface DashboardtypesDashboardSpecDTO {
/**

View File

@@ -2,16 +2,15 @@ import { useState } from 'react';
import { Button } from '@signozhq/ui/button';
import { Typography } from '@signozhq/ui/typography';
import dashboardVariablesQuery from 'api/dashboard/variables/dashboardVariablesQuery';
import { DashboardtypesListVariableSpecSortDTO as VariableSortDTO } from 'api/generated/services/sigNoz.schemas';
import Editor from 'components/Editor';
import sortValues from 'lib/dashboardVariables/sortVariableValues';
import { sortDirectionOf } from '../variableModel';
import type { VariableSort } from '../variableModel';
import styles from './VariableForm.module.scss';
interface QueryVariableFieldsProps {
queryValue: string;
sort: VariableSortDTO;
sort: VariableSort;
onChange: (queryValue: string) => void;
onPreview: (values: (string | number)[]) => void;
onError: (message: string | null) => void;
@@ -37,10 +36,7 @@ function QueryVariableFields({
});
if (res.statusCode === 200 && res.payload) {
onPreview(
sortValues(res.payload.variableValues ?? [], sortDirectionOf(sort)) as (
| string
| number
)[],
sortValues(res.payload.variableValues ?? [], sort) as (string | number)[],
);
} else {
onError(res.error || 'Failed to run query');

View File

@@ -12,12 +12,10 @@ import { Collapse, Input as AntdInput, Select } from 'antd';
import { commaValuesParser } from 'lib/dashboardVariables/customCommaValuesParser';
import sortValues from 'lib/dashboardVariables/sortVariableValues';
import { DashboardtypesListVariableSpecSortDTO as VariableSortDTO } from 'api/generated/services/sigNoz.schemas';
import {
sortDirectionOf,
VARIABLE_SORTS,
type VariableFormModel,
type VariableSort,
type VariableType,
} from '../variableModel';
import DynamicVariableFields from './DynamicVariableFields';
@@ -25,16 +23,10 @@ import QueryVariableFields from './QueryVariableFields';
import VariableTypeSelector from './VariableTypeSelector';
import styles from './VariableForm.module.scss';
const SORT_LABEL: Record<VariableSortDTO, string> = {
[VariableSortDTO.none]: 'Disabled',
[VariableSortDTO['alphabetical-asc']]: 'Alphabetical (asc)',
[VariableSortDTO['alphabetical-desc']]: 'Alphabetical (desc)',
[VariableSortDTO['numerical-asc']]: 'Numerical (asc)',
[VariableSortDTO['numerical-desc']]: 'Numerical (desc)',
[VariableSortDTO['alphabetical-ci-asc']]:
'Alphabetical, case-insensitive (asc)',
[VariableSortDTO['alphabetical-ci-desc']]:
'Alphabetical, case-insensitive (desc)',
const SORT_LABEL: Record<VariableSort, string> = {
DISABLED: 'Disabled',
ASC: 'Ascending',
DESC: 'Descending',
};
function getNameError(name: string, existingNames: string[]): string | null {
@@ -99,10 +91,7 @@ function VariableForm({
const onCustomChange = (value: string): void => {
set({ customValue: value });
setPreviewValues(
sortValues(commaValuesParser(value), sortDirectionOf(model.sort)) as (
| string
| number
)[],
sortValues(commaValuesParser(value), model.sort) as (string | number)[],
);
};
@@ -270,7 +259,7 @@ function VariableForm({
label: SORT_LABEL[sort],
value: sort,
}))}
onChange={(value): void => set({ sort: value as VariableSortDTO })}
onChange={(value): void => set({ sort: value as VariableSort })}
testId="variable-sort-select"
/>
</div>

View File

@@ -1,17 +1,16 @@
import {
DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDashboardtypesTextVariableSpecDTOKind as TextEnvelopeKind,
DashboardtypesVariableEnvelopeGithubComPersesSpecGoDashboardTextVariableSpecDTOKind as TextEnvelopeKind,
DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDashboardtypesListVariableSpecDTOKind as ListEnvelopeKind,
DashboardtypesVariablePluginVariantGithubComSigNozSignozPkgTypesDashboardtypesCustomVariableSpecDTOKind as CustomPluginKind,
DashboardtypesVariablePluginVariantGithubComSigNozSignozPkgTypesDashboardtypesDynamicVariableSpecDTOKind as DynamicPluginKind,
DashboardtypesVariablePluginVariantGithubComSigNozSignozPkgTypesDashboardtypesQueryVariableSpecDTOKind as QueryPluginKind,
DashboardtypesListVariableSpecSortDTO as VariableSortDTO,
TelemetrytypesSignalDTO,
} from 'api/generated/services/sigNoz.schemas';
import type {
DashboardtypesListVariableSpecDTO,
DashboardtypesVariableDTO,
DashboardtypesVariablePluginDTO,
DashboardtypesTextVariableSpecDTO,
DashboardTextVariableSpecDTO,
} from 'api/generated/services/sigNoz.schemas';
import {
@@ -19,6 +18,7 @@ import {
PLUGIN_KIND,
type TelemetrySignal,
type VariableFormModel,
type VariableSort,
} from './variableModel';
/** DTO envelope → flat form model (for display / editing). */
@@ -35,7 +35,7 @@ export function dtoToFormModel(
// Text variable — a distinct envelope (no list plugin).
if (dto.kind === TextEnvelopeKind.TextVariable) {
const spec = dto.spec as DashboardtypesTextVariableSpecDTO;
const spec = dto.spec as DashboardTextVariableSpecDTO;
return {
...common,
type: 'TEXT',
@@ -50,7 +50,7 @@ export function dtoToFormModel(
...common,
multiSelect: spec.allowMultiple ?? false,
showAllOption: spec.allowAllValue ?? false,
sort: spec.sort ?? VariableSortDTO.none,
sort: (spec.sort as VariableSort) ?? 'DISABLED',
defaultValue: spec.defaultValue,
};
const plugin = spec.plugin;

View File

@@ -1,6 +1,4 @@
import { DashboardtypesListVariableSpecSortDTO as VariableSortDTO } from 'api/generated/services/sigNoz.schemas';
import type { VariableDefaultValueDTO } from 'api/generated/services/sigNoz.schemas';
import type { TSortVariableValuesType } from 'types/api/dashboard/getAll';
/**
* Flat, UI-friendly representation of a V2 dashboard variable. The wire format
@@ -10,6 +8,8 @@ import type { TSortVariableValuesType } from 'types/api/dashboard/getAll';
export type VariableType = 'QUERY' | 'CUSTOM' | 'TEXT' | 'DYNAMIC';
export type VariableSort = 'DISABLED' | 'ASC' | 'DESC';
export type TelemetrySignal = 'traces' | 'logs' | 'metrics';
/** Wire `kind` discriminators (string values of the generated enums). */
@@ -24,20 +24,7 @@ export const PLUGIN_KIND = {
DYNAMIC: 'signoz/DynamicVariable',
} as const;
export const VARIABLE_SORTS: VariableSortDTO[] = Object.values(VariableSortDTO);
/** Direction the preview sorter should apply for a given wire sort value. */
export function sortDirectionOf(
sort: VariableSortDTO,
): TSortVariableValuesType {
if (sort.endsWith('-asc')) {
return 'ASC';
}
if (sort.endsWith('-desc')) {
return 'DESC';
}
return 'DISABLED';
}
export const VARIABLE_SORTS: VariableSort[] = ['DISABLED', 'ASC', 'DESC'];
export const TELEMETRY_SIGNALS: TelemetrySignal[] = [
'traces',
@@ -55,7 +42,7 @@ export interface VariableFormModel {
// List-variable common fields (Query / Custom / Dynamic).
multiSelect: boolean;
showAllOption: boolean;
sort: VariableSortDTO;
sort: VariableSort;
// Type-specific.
queryValue: string; // QUERY
@@ -80,7 +67,7 @@ export function emptyVariableFormModel(): VariableFormModel {
type: 'QUERY',
multiSelect: false,
showAllOption: false,
sort: VariableSortDTO.none,
sort: 'DISABLED',
queryValue: '',
customValue: '',
textValue: '',

View File

@@ -3,15 +3,16 @@ package flagger
import "github.com/SigNoz/signoz/pkg/types/featuretypes"
var (
FeatureUseSpanMetrics = featuretypes.MustNewName("use_span_metrics")
FeatureKafkaSpanEval = featuretypes.MustNewName("kafka_span_eval")
FeatureHideRootUser = featuretypes.MustNewName("hide_root_user")
FeatureGetMetersFromZeus = featuretypes.MustNewName("get_meters_from_zeus")
FeaturePutMetersInZeus = featuretypes.MustNewName("put_meters_in_zeus")
FeatureUseMeterReporter = featuretypes.MustNewName("use_meter_reporter")
FeatureUseJSONBody = featuretypes.MustNewName("use_json_body")
FeatureUseFineGrainedAuthz = featuretypes.MustNewName("use_fine_grained_authz")
FeatureUseDashboardV2 = featuretypes.MustNewName("use_dashboard_v2")
FeatureUseSpanMetrics = featuretypes.MustNewName("use_span_metrics")
FeatureKafkaSpanEval = featuretypes.MustNewName("kafka_span_eval")
FeatureHideRootUser = featuretypes.MustNewName("hide_root_user")
FeatureGetMetersFromZeus = featuretypes.MustNewName("get_meters_from_zeus")
FeaturePutMetersInZeus = featuretypes.MustNewName("put_meters_in_zeus")
FeatureUseMeterReporter = featuretypes.MustNewName("use_meter_reporter")
FeatureUseJSONBody = featuretypes.MustNewName("use_json_body")
FeatureUseFineGrainedAuthz = featuretypes.MustNewName("use_fine_grained_authz")
FeatureUseDashboardV2 = featuretypes.MustNewName("use_dashboard_v2")
FeatureEnableMetricsReduction = featuretypes.MustNewName("enable_metrics_reduction")
)
func MustNewRegistry() featuretypes.Registry {
@@ -88,6 +89,14 @@ func MustNewRegistry() featuretypes.Registry {
DefaultVariant: featuretypes.MustNewName("disabled"),
Variants: featuretypes.NewBooleanVariants(),
},
&featuretypes.Feature{
Name: FeatureEnableMetricsReduction,
Kind: featuretypes.KindBoolean,
Stage: featuretypes.StageExperimental,
Description: "Controls whether metrics cardinality reduction (buffer/reduced tables) is read by the querier",
DefaultVariant: featuretypes.MustNewName("disabled"),
Variants: featuretypes.NewBooleanVariants(),
},
)
if err != nil {
panic(err)

View File

@@ -341,12 +341,12 @@ func alignedMetricWindow(startMs, endMs int64) (
}
tsAdjustedStartMs, _, distributedTSTable, localTSTable := telemetrymetrics.WhichTSTableToUse(
samplesAdjustedStartMs, flooredEndMs, nil,
samplesAdjustedStartMs, flooredEndMs, false, nil,
)
distributedSamplesTable, localSamplesTable := telemetrymetrics.WhichSamplesTableToUse(
samplesAdjustedStartMs, flooredEndMs,
metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, nil,
metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil,
)
return samplesAdjustedStartMs, flooredEndMs, tsAdjustedStartMs, distributedTSTable, localTSTable, distributedSamplesTable, localSamplesTable

View File

@@ -141,7 +141,7 @@ func (m *module) listMetrics(ctx context.Context, orgID valuer.UUID, params *met
sb.Select("DISTINCT metric_name")
if params.Start != nil && params.End != nil {
start, end, distributedTsTable, _ := telemetrymetrics.WhichTSTableToUse(uint64(*params.Start), uint64(*params.End), nil)
start, end, distributedTsTable, _ := telemetrymetrics.WhichTSTableToUse(uint64(*params.Start), uint64(*params.End), false, nil)
sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
sb.Where(sb.Between("unix_milli", start, end))
} else {
@@ -527,7 +527,7 @@ func (m *module) InspectMetrics(
return nil, err
}
tsStart, _, tsTable, _ := telemetrymetrics.WhichTSTableToUse(start, end, nil)
tsStart, _, tsTable, _ := telemetrymetrics.WhichTSTableToUse(start, end, false, nil)
tsSb := sqlbuilder.NewSelectBuilder()
tsSb.Select("fingerprint", "labels")
tsSb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, tsTable))
@@ -971,8 +971,8 @@ func (m *module) fetchMetricsStatsWithSamples(
}
}
start, end, distributedTsTable, localTsTable := telemetrymetrics.WhichTSTableToUse(uint64(req.Start), uint64(req.End), nil)
distributedSamplesTable, _ := telemetrymetrics.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, nil)
start, end, distributedTsTable, localTsTable := telemetrymetrics.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
distributedSamplesTable, _ := telemetrymetrics.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil)
countExp := telemetrymetrics.CountExpressionForSamplesTable(distributedSamplesTable)
// Timeseries counts per metric
@@ -1100,7 +1100,7 @@ func (m *module) computeTimeseriesTreemap(ctx context.Context, req *metricsexplo
}
}
start, end, distributedTsTable, _ := telemetrymetrics.WhichTSTableToUse(uint64(req.Start), uint64(req.End), nil)
start, end, distributedTsTable, _ := telemetrymetrics.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
totalTSBuilder := sqlbuilder.NewSelectBuilder()
totalTSBuilder.Select("uniq(fingerprint) AS total_time_series")
@@ -1176,8 +1176,8 @@ func (m *module) computeSamplesTreemap(ctx context.Context, req *metricsexplorer
}
}
start, end, distributedTsTable, localTsTable := telemetrymetrics.WhichTSTableToUse(uint64(req.Start), uint64(req.End), nil)
distributedSamplesTable, _ := telemetrymetrics.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, nil)
start, end, distributedTsTable, localTsTable := telemetrymetrics.WhichTSTableToUse(uint64(req.Start), uint64(req.End), false, nil)
distributedSamplesTable, _ := telemetrymetrics.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, false, nil)
countExp := telemetrymetrics.CountExpressionForSamplesTable(distributedSamplesTable)
candidateLimit := req.Limit + 50

View File

@@ -91,13 +91,22 @@ func (q *builderQuery[T]) Fingerprint() string {
if a.ComparisonSpaceAggregationParam != nil {
spaceAggParamStr = a.ComparisonSpaceAggregationParam.StringValue()
}
aggParts = append(aggParts, fmt.Sprintf("%s:%s:%s:%s:%s",
part := fmt.Sprintf("%s:%s:%s:%s:%s",
a.MetricName,
a.Temporality.StringValue(),
a.TimeAggregation.StringValue(),
a.SpaceAggregation.StringValue(),
spaceAggParamStr,
))
)
if a.Reduced {
oneDay := uint64(24 * time.Hour.Milliseconds())
route := "reduced"
if q.toMS-q.fromMS < oneDay && q.fromMS >= uint64(time.Now().UnixMilli())-oneDay {
route = "buffer"
}
part += ":" + route
}
aggParts = append(aggParts, part)
}
}
parts = append(parts, fmt.Sprintf("aggs=[%s]", strings.Join(aggParts, ",")))

View File

@@ -119,7 +119,7 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
queries := make(map[string]qbtypes.Query)
steps := make(map[string]qbtypes.Step)
missingMetricQueries, metricWarnings, err := q.resolveMetricMetadata(ctx, req.CompositeQuery.Queries, req.Start, req.End)
missingMetricQueries, metricWarnings, err := q.resolveMetricMetadata(ctx, orgID, req.CompositeQuery.Queries, req.Start, req.End)
if err != nil {
return nil, err
}
@@ -304,7 +304,7 @@ func (q *querier) populateQBEvent(event *qbtypes.QBEvent, queries []qbtypes.Quer
// resolved: never-seen metrics and dormant metrics (seen but no data in
// the query window).
// - err: Internal when a metadata fetch fails.
func (q *querier) resolveMetricMetadata(ctx context.Context, queries []qbtypes.QueryEnvelope, start, end uint64) (missingMetricQueries []string, metricWarnings []string, err error) {
func (q *querier) resolveMetricMetadata(ctx context.Context, orgID valuer.UUID, queries []qbtypes.QueryEnvelope, start, end uint64) (missingMetricQueries []string, metricWarnings []string, err error) {
metricNames := make([]string, 0)
for idx := range queries {
if queries[idx].Type != qbtypes.QueryTypeBuilder {
@@ -325,7 +325,7 @@ func (q *querier) resolveMetricMetadata(ctx context.Context, queries []qbtypes.Q
return nil, nil, nil
}
metricTemporality, metricTypes, err := q.metadataStore.FetchTemporalityAndTypeMulti(ctx, start, end, metricNames...)
metricTemporality, metricTypes, reducedMetricsSet, err := q.metadataStore.FetchTemporalityAndTypeMulti(ctx, orgID, start, end, metricNames...)
if err != nil {
q.logger.WarnContext(ctx, "failed to fetch metric temporality", errors.Attr(err), slog.Any("metrics", metricNames))
return nil, nil, errors.NewInternalf(errors.CodeInternal, "failed to fetch metrics temporality")
@@ -362,6 +362,9 @@ func (q *querier) resolveMetricMetadata(ctx context.Context, queries []qbtypes.Q
if err := spec.Aggregations[i].ValidateForType(); err != nil {
return nil, nil, err
}
if reducedMetricsSet[spec.Aggregations[i].MetricName] {
spec.Aggregations[i].Reduced = true
}
presentAggregations = append(presentAggregations, spec.Aggregations[i])
}
if len(presentAggregations) == 0 {

View File

@@ -2136,12 +2136,12 @@ func (t *telemetryMetaStore) GetAllValues(ctx context.Context, fieldValueSelecto
return values, complete, nil
}
func (t *telemetryMetaStore) FetchTemporality(ctx context.Context, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricName string) (metrictypes.Temporality, error) {
func (t *telemetryMetaStore) FetchTemporality(ctx context.Context, orgID valuer.UUID, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricName string) (metrictypes.Temporality, error) {
if metricName == "" {
return metrictypes.Unknown, errors.Newf(errors.TypeInternal, errors.CodeInternal, "metric name cannot be empty")
}
temporalityMap, err := t.FetchTemporalityMulti(ctx, queryTimeRangeStartTs, queryTimeRangeEndTs, metricName)
temporalityMap, err := t.FetchTemporalityMulti(ctx, orgID, queryTimeRangeStartTs, queryTimeRangeEndTs, metricName)
if err != nil {
return metrictypes.Unknown, err
}
@@ -2154,25 +2154,27 @@ func (t *telemetryMetaStore) FetchTemporality(ctx context.Context, queryTimeRang
return temporality, nil
}
func (t *telemetryMetaStore) FetchTemporalityMulti(ctx context.Context, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricNames ...string) (map[string]metrictypes.Temporality, error) {
temporalities, _, err := t.FetchTemporalityAndTypeMulti(ctx, queryTimeRangeStartTs, queryTimeRangeEndTs, metricNames...)
func (t *telemetryMetaStore) FetchTemporalityMulti(ctx context.Context, orgID valuer.UUID, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricNames ...string) (map[string]metrictypes.Temporality, error) {
temporalities, _, _, err := t.FetchTemporalityAndTypeMulti(ctx, orgID, queryTimeRangeStartTs, queryTimeRangeEndTs, metricNames...)
return temporalities, err
}
func (t *telemetryMetaStore) FetchTemporalityAndTypeMulti(ctx context.Context, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricNames ...string) (map[string]metrictypes.Temporality, map[string]metrictypes.Type, error) {
func (t *telemetryMetaStore) FetchTemporalityAndTypeMulti(ctx context.Context, orgID valuer.UUID, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricNames ...string) (map[string]metrictypes.Temporality, map[string]metrictypes.Type, map[string]bool, error) {
if len(metricNames) == 0 {
return make(map[string]metrictypes.Temporality), make(map[string]metrictypes.Type), nil
return make(map[string]metrictypes.Temporality), make(map[string]metrictypes.Type), make(map[string]bool), nil
}
reductionEnabled := t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID))
temporalities := make(map[string]metrictypes.Temporality)
types := make(map[string]metrictypes.Type)
metricsTemporality, metricTypes, err := t.fetchMetricsTemporalityAndType(ctx, queryTimeRangeStartTs, queryTimeRangeEndTs, metricNames...)
metricsTemporality, metricTypes, reduced, err := t.fetchMetricsTemporalityAndType(ctx, queryTimeRangeStartTs, queryTimeRangeEndTs, reductionEnabled, metricNames...)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
meterMetricsTemporality, meterMetricsTypes, err := t.fetchMeterSourceMetricsTemporalityAndType(ctx, metricNames...)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
// For metrics not found in the database, set to Unknown
@@ -2197,10 +2199,10 @@ func (t *telemetryMetaStore) FetchTemporalityAndTypeMulti(ctx context.Context, q
}
}
return temporalities, types, nil
return temporalities, types, reduced, nil
}
func (t *telemetryMetaStore) fetchMetricsTemporalityAndType(ctx context.Context, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricNames ...string) (map[string][]metrictypes.Temporality, map[string]metrictypes.Type, error) {
func (t *telemetryMetaStore) fetchMetricsTemporalityAndType(ctx context.Context, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, reductionEnabled bool, metricNames ...string) (map[string][]metrictypes.Temporality, map[string]metrictypes.Type, map[string]bool, error) {
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalMetrics.StringValue(),
instrumentationtypes.CodeNamespace: "metadata",
@@ -2208,48 +2210,58 @@ func (t *telemetryMetaStore) fetchMetricsTemporalityAndType(ctx context.Context,
})
temporalities := make(map[string][]metrictypes.Temporality)
types := make(map[string]metrictypes.Type)
reduced := make(map[string]bool)
adjustedStartTs, adjustedEndTs, tsTableName, _ := telemetrymetrics.WhichTSTableToUse(queryTimeRangeStartTs, queryTimeRangeEndTs, nil)
adjustedStartTs, adjustedEndTs, tsTableName, _ := telemetrymetrics.WhichTSTableToUse(queryTimeRangeStartTs, queryTimeRangeEndTs, false, nil)
// Build query to fetch temporality for all metrics
// We use attr_string_value where attr_name = '__temporality__'
// Note: The columns are mixed in the current data - temporality column contains metric_name
// and metric_name column contains temporality value, so we use the correct mapping
sb := sqlbuilder.Select(
"metric_name",
"temporality",
"any(type) AS type",
"any(is_monotonic) as is_monotonic",
).
From(t.metricsDBName + "." + tsTableName)
cols := []string{"metric_name", "temporality", "any(type) AS type", "any(is_monotonic) as is_monotonic"}
// Filter by metric names (in the temporality column due to data mix-up)
// When reduction is enabled, fold the reduced-catalog presence check into the
// same query so a metric's reduced status comes back in one round trip.
var reducedArgs []any
if reductionEnabled {
rs := sqlbuilder.NewSelectBuilder()
rs.Select("metric_name")
rs.From(t.metricsDBName + "." + telemetrymetrics.TimeseriesV4ReducedTableName)
rs.Where(rs.In("metric_name", metricNames), rs.GTE("unix_milli", adjustedStartTs), rs.LT("unix_milli", adjustedEndTs))
rs.GroupBy("metric_name")
rsQuery, rsArgs := rs.BuildWithFlavor(sqlbuilder.ClickHouse)
cols = append(cols, fmt.Sprintf("metric_name GLOBAL IN (%s) AS reduced", rsQuery))
reducedArgs = rsArgs
}
sb := sqlbuilder.NewSelectBuilder()
sb.Select(cols...)
sb.From(t.metricsDBName + "." + tsTableName)
sb.Where(
sb.In("metric_name", metricNames),
sb.GTE("unix_milli", adjustedStartTs),
sb.LT("unix_milli", adjustedEndTs),
)
sb.GroupBy("metric_name", "temporality")
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse, reducedArgs...)
t.logger.DebugContext(ctx, "fetching metric temporality", slog.String("query", query), slog.Any("args", args))
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
if err != nil {
return nil, nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "failed to fetch metric temporality")
return nil, nil, nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "failed to fetch metric temporality")
}
defer rows.Close()
// Process results
for rows.Next() {
var metricName string
var temporality metrictypes.Temporality
var metricType metrictypes.Type
var isMonotonic bool
if err := rows.Scan(&metricName, &temporality, &metricType, &isMonotonic); err != nil {
return nil, nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "failed to scan temporality result")
var isReduced uint8
dest := []any{&metricName, &temporality, &metricType, &isMonotonic}
if reductionEnabled {
dest = append(dest, &isReduced)
}
if err := rows.Scan(dest...); err != nil {
return nil, nil, nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "failed to scan temporality result")
}
if temporality != metrictypes.Unknown {
temporalities[metricName] = append(temporalities[metricName], temporality)
@@ -2258,12 +2270,15 @@ func (t *telemetryMetaStore) fetchMetricsTemporalityAndType(ctx context.Context,
metricType = metrictypes.GaugeType
}
types[metricName] = metricType
if isReduced != 0 {
reduced[metricName] = true
}
}
if err := rows.Err(); err != nil {
return nil, nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "error iterating over metrics temporality rows")
return nil, nil, nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "error iterating over metrics temporality rows")
}
return temporalities, types, nil
return temporalities, types, reduced, nil
}
func (t *telemetryMetaStore) fetchMeterSourceMetricsTemporalityAndType(ctx context.Context, metricNames ...string) (map[string]metrictypes.Temporality, map[string]metrictypes.Type, error) {

View File

@@ -0,0 +1,157 @@
package telemetrymetrics
import (
"context"
"testing"
"time"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
"github.com/stretchr/testify/require"
)
func reducedQuery(metric string, ty metrictypes.Type, temp metrictypes.Temporality, ta metrictypes.TimeAggregation, sa metrictypes.SpaceAggregation) qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation] {
return qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
StepInterval: qbtypes.Step{Duration: 5 * time.Minute},
Aggregations: []qbtypes.MetricAggregation{{
MetricName: metric,
Type: ty,
Temporality: temp,
TimeAggregation: ta,
SpaceAggregation: sa,
Reduced: true,
}},
}
}
func TestReducedStatementBuilder(t *testing.T) {
cases := []struct {
name string
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]
expected qbtypes.Statement
}{
{
name: "gauge_sum_latest",
query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationLatest, metrictypes.SpaceAggregationSum),
expected: qbtypes.Statement{
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, anyLast(last) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, argMax(value, unix_milli) AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum_last`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.distributed_time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
},
},
{
name: "gauge_avg_avg",
query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationAvg, metrictypes.SpaceAggregationAvg),
expected: qbtypes.Statement{
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(sum) / sum(count) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, avg(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum_last`, computed_at) AS value, argMax(`count_series`, computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.distributed_time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
},
},
{
name: "gauge_min_min",
query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationMin, metrictypes.SpaceAggregationMin),
expected: qbtypes.Statement{
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, min(min) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, min(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, min(value) AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`min`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.distributed_time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, min(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
},
},
{
name: "gauge_max_max",
query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationMax, metrictypes.SpaceAggregationMax),
expected: qbtypes.Statement{
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, max(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(value) AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`max`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.distributed_time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, max(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
},
},
{
name: "counter_sum_rate",
query: reducedQuery("test.metric", metrictypes.SumType, metrictypes.Cumulative, metrictypes.TimeAggregationRate, metrictypes.SpaceAggregationSum),
expected: qbtypes.Statement{
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(value) / 300 AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.distributed_time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric", uint64(1746999600000), uint64(1747172760000), 0, "test.metric", uint64(1746999600000), uint64(1747172760000), "test.metric", uint64(1746999600000), uint64(1747172760000), false},
},
},
{
name: "counter_avg_increase",
query: reducedQuery("test.metric", metrictypes.SumType, metrictypes.Cumulative, metrictypes.TimeAggregationIncrease, metrictypes.SpaceAggregationAvg),
expected: qbtypes.Statement{
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value, per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum`, computed_at) AS value, argMax(`count_series`, computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.distributed_time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric", uint64(1746999600000), uint64(1747172760000), 0, "test.metric", uint64(1746999600000), uint64(1747172760000), "test.metric", uint64(1746999600000), uint64(1747172760000), false},
},
},
{
name: "counter_min_omitted",
query: reducedQuery("test.metric", metrictypes.SumType, metrictypes.Cumulative, metrictypes.TimeAggregationRate, metrictypes.SpaceAggregationMin),
expected: qbtypes.Statement{
Query: "WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, min(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric", uint64(1746999600000), uint64(1747172760000), 0},
},
},
{
name: "counter_max_omitted",
query: reducedQuery("test.metric", metrictypes.SumType, metrictypes.Cumulative, metrictypes.TimeAggregationRate, metrictypes.SpaceAggregationMax),
expected: qbtypes.Statement{
Query: "WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, max(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric", uint64(1746999600000), uint64(1747172760000), 0},
},
},
{
name: "histogram_p99",
query: reducedQuery("test.metric", metrictypes.HistogramType, metrictypes.Cumulative, metrictypes.TimeAggregationUnspecified, metrictypes.SpaceAggregationPercentile99),
expected: qbtypes.Statement{
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, sum(value) / 300 AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.distributed_time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts, `le`), __spatial_aggregation_cte AS (SELECT ts, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
},
},
{
name: "summary_avg",
query: reducedQuery("test.metric", metrictypes.SummaryType, metrictypes.Unspecified, metrictypes.TimeAggregationAvg, metrictypes.SpaceAggregationAvg),
expected: qbtypes.Statement{
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(sum) / sum(count) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, avg(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum_last`, computed_at) AS value, argMax(`count_series`, computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.distributed_time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
},
},
}
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
fl, err := flagger.New(context.Background(), instrumentationtest.New().ToProviderSettings(), flagger.Config{}, flagger.MustNewRegistry())
require.NoError(t, err)
sb := NewMetricQueryStatementBuilder(instrumentationtest.New().ToProviderSettings(), telemetrytypestest.NewMockMetadataStore(), fm, cb, fl)
const start, end = uint64(1747000000000), uint64(1747172800000)
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, err := sb.Build(context.Background(), start, end, qbtypes.RequestTypeTimeSeries, c.query, nil)
require.NoError(t, err)
require.Equal(t, c.expected.Query, got.Query)
require.Equal(t, c.expected.Args, got.Args)
})
}
t.Run("buffer_recent_window", func(t *testing.T) {
now := time.Now().UnixMilli()
q := reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationLatest, metrictypes.SpaceAggregationSum)
got, err := sb.Build(context.Background(), uint64(now-2*time.Hour.Milliseconds()), uint64(now), qbtypes.RequestTypeTimeSeries, q, nil)
require.NoError(t, err)
require.Contains(t, got.Query, "signoz_metrics.distributed_samples_v4_buffer")
require.Contains(t, got.Query, "signoz_metrics.time_series_v4_buffer")
require.Contains(t, got.Query, "is_reduced")
require.NotContains(t, got.Query, "UNION ALL")
})
t.Run("not_reduced", func(t *testing.T) {
q := reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationLatest, metrictypes.SpaceAggregationSum)
q.Aggregations[0].Reduced = false
got, err := sb.Build(context.Background(), start, end, qbtypes.RequestTypeTimeSeries, q, nil)
require.NoError(t, err)
require.NotContains(t, got.Query, "UNION ALL")
require.NotContains(t, got.Query, "reduced")
require.NotContains(t, got.Query, "buffer")
})
}

View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log/slog"
"time"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
@@ -180,19 +181,30 @@ func (b *MetricQueryStatementBuilder) buildPipelineStatement(
query.Aggregations[0].SpaceAggregation = metrictypes.SpaceAggregationSum
}
agg := query.Aggregations[0]
// A reduced metric reads the raw buffer for recent short windows, and
// samples_v4/agg (unioned with the reduced tables) otherwise. The buffer is
// shaped exactly like samples_v4 / time_series_v4, so once the table names are
// chosen the rest of the pipeline is unchanged.
useBuffer := agg.Reduced &&
end-start < oneDayInMilliseconds &&
start >= uint64(time.Now().UnixMilli())-oneDayInMilliseconds
samplesTable, _ := WhichSamplesTableToUse(start, end, agg.Type, agg.TimeAggregation, useBuffer, agg.TableHints)
tsStart, tsEnd, _, tsTable := WhichTSTableToUse(start, end, useBuffer, agg.TableHints)
var timeSeriesCTE string
var timeSeriesCTEArgs []any
var err error
// time_series_cte
// this is applicable for all the queries
if timeSeriesCTE, timeSeriesCTEArgs, err = b.buildTimeSeriesCTE(ctx, start, end, query, keys, variables); err != nil {
if timeSeriesCTE, timeSeriesCTEArgs, err = b.buildTimeSeriesCTE(ctx, tsStart, tsEnd, query, keys, variables, tsTable); err != nil {
return nil, err
}
if qbtypes.CanShortCircuitDelta(query.Aggregations[0]) {
// spatial_aggregation_cte directly for certain delta queries
if frag, args, err := b.buildTemporalAggDeltaFastPath(start, end, query, timeSeriesCTE, timeSeriesCTEArgs); err != nil {
if frag, args, err := b.buildTemporalAggDeltaFastPath(start, end, query, samplesTable, timeSeriesCTE, timeSeriesCTEArgs); err != nil {
return nil, err
} else if frag != "" {
cteFragments = append(cteFragments, frag)
@@ -200,7 +212,7 @@ func (b *MetricQueryStatementBuilder) buildPipelineStatement(
}
} else {
// temporal_aggregation_cte
if frag, args, err := b.buildTemporalAggregationCTE(ctx, start, end, query, keys, timeSeriesCTE, timeSeriesCTEArgs); err != nil {
if frag, args, err := b.buildTemporalAggregationCTE(ctx, start, end, query, keys, samplesTable, timeSeriesCTE, timeSeriesCTEArgs); err != nil {
return nil, err
} else if frag != "" {
cteFragments = append(cteFragments, frag)
@@ -214,18 +226,188 @@ func (b *MetricQueryStatementBuilder) buildPipelineStatement(
}
}
var reducedFragments []string
var reducedArgs [][]any
if agg.Reduced && !useBuffer {
var tsCTE string
var tsArgs []any
if tsCTE, tsArgs, err = b.buildReducedTimeSeriesCTE(ctx, start, end, query, keys, variables); err != nil {
return nil, err
}
if temporalFrag, temporalArgs, ok := b.buildReducedTemporalAggregationCTE(start, end, query, tsCTE, tsArgs); ok {
spatialFrag, spatialArgs := b.buildReducedSpatialAggregationCTE(query)
reducedFragments = []string{temporalFrag, spatialFrag}
reducedArgs = [][]any{temporalArgs, spatialArgs}
}
}
// reset the query to the original state
query.Aggregations[0].SpaceAggregation = origSpaceAgg
query.Aggregations[0].TimeAggregation = origTimeAgg
query.GroupBy = origGroupBy
// final SELECT
return b.BuildFinalSelect(cteFragments, cteArgs, query)
mainStmt, err := b.BuildFinalSelect(cteFragments, cteArgs, query)
if err != nil {
return nil, err
}
if reducedFragments == nil {
return mainStmt, nil
}
reducedStmt, err := b.BuildFinalSelect(reducedFragments, reducedArgs, query)
if err != nil {
return nil, err
}
return unionStatements(mainStmt, reducedStmt, query)
}
func unionStatements(main, reduced *qbtypes.Statement, query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]) (*qbtypes.Statement, error) {
orderBy := "ts"
for _, g := range query.GroupBy {
orderBy = fmt.Sprintf("`%s`, ", g.Name) + orderBy
}
q := fmt.Sprintf("SELECT * FROM (%s) UNION ALL SELECT * FROM (%s) ORDER BY %s", main.Query, reduced.Query, orderBy)
args := append(append([]any{}, main.Args...), reduced.Args...)
warnings := append(append([]string{}, main.Warnings...), reduced.Warnings...)
return &qbtypes.Statement{Query: q, Args: args, Warnings: warnings}, nil
}
func (b *MetricQueryStatementBuilder) buildReducedTimeSeriesCTE(
ctx context.Context,
start, end uint64,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
) (string, []any, error) {
sb := sqlbuilder.NewSelectBuilder()
var preparedWhereClause querybuilder.PreparedWhereClause
var err error
if query.Filter != nil && query.Filter.Expression != "" {
preparedWhereClause, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
Context: ctx,
Logger: b.logger,
FieldMapper: b.fm,
ConditionBuilder: b.cb,
FieldKeys: keys,
FullTextColumn: &telemetrytypes.TelemetryFieldKey{Name: "labels"},
Variables: variables,
StartNs: start,
EndNs: end,
})
if err != nil {
return "", nil, err
}
}
sb.From(fmt.Sprintf("%s.%s", DBName, TimeseriesV4ReducedTableName))
sb.Select("fingerprint")
for _, g := range query.GroupBy {
col, err := b.fm.ColumnExpressionFor(ctx, start, end, &g.TelemetryFieldKey, keys)
if err != nil {
return "", nil, err
}
sb.SelectMore(col)
}
sb.Where(
sb.In("metric_name", query.Aggregations[0].MetricName),
sb.GTE("unix_milli", start),
sb.LTE("unix_milli", end),
sb.EQ("__normalized", false),
)
if !preparedWhereClause.IsEmpty() {
sb.AddWhereClause(preparedWhereClause.WhereClause)
}
sb.GroupBy("fingerprint")
sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)
q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
return fmt.Sprintf("(%s) AS filtered_time_series", q), args, nil
}
func (b *MetricQueryStatementBuilder) buildReducedTemporalAggregationCTE(
start, end uint64,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
timeSeriesCTE string,
timeSeriesCTEArgs []any,
) (string, []any, bool) {
agg := query.Aggregations[0]
stepSec := int64(query.StepInterval.Seconds())
value, weight, ok := ReducedValueColumn(agg.Type, agg.SpaceAggregation)
if !ok {
return "", nil, false
}
// dedup recomputed buckets: latest computed_at wins per (series, 60s bucket)
dedup := sqlbuilder.NewSelectBuilder()
dedup.Select("reduced_fingerprint AS fingerprint", "unix_milli")
dedup.SelectMore(fmt.Sprintf("argMax(%s, computed_at) AS value", value))
if weight != "" {
dedup.SelectMore(fmt.Sprintf("argMax(%s, computed_at) AS weight", weight))
}
dedup.From(fmt.Sprintf("%s.%s", DBName, WhichReducedSamplesTableToUse(agg.Type)))
dedup.Where(
dedup.In("metric_name", agg.MetricName),
dedup.GTE("unix_milli", start),
dedup.LT("unix_milli", end),
)
dedup.GroupBy("reduced_fingerprint", "unix_milli")
dedupQuery, dedupArgs := dedup.BuildWithFlavor(sqlbuilder.ClickHouse)
sb := sqlbuilder.NewSelectBuilder()
sb.Select("fingerprint")
sb.SelectMore(fmt.Sprintf("toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(%d)) AS ts", stepSec))
for _, g := range query.GroupBy {
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
}
sb.SelectMore(fmt.Sprintf("%s AS per_series_value", ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec)))
if weight != "" {
// count_series is a series count, not additive over time, so the avg
// denominator is reduced with avg
sb.SelectMore("avg(weight) AS per_series_weight")
}
sb.From(fmt.Sprintf("(%s) AS points", dedupQuery))
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
sb.GroupBy("fingerprint", "ts")
sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)
initArgs := append(append([]any{}, dedupArgs...), timeSeriesCTEArgs...)
q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse, initArgs...)
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, true
}
func (b *MetricQueryStatementBuilder) buildReducedSpatialAggregationCTE(
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
) (string, []any) {
spatial := "sum(per_series_value)"
switch query.Aggregations[0].SpaceAggregation {
case metrictypes.SpaceAggregationAvg:
spatial = "sum(per_series_value) / sum(per_series_weight)"
case metrictypes.SpaceAggregationMin:
spatial = "min(per_series_value)"
case metrictypes.SpaceAggregationMax:
spatial = "max(per_series_value)"
}
sb := sqlbuilder.NewSelectBuilder()
sb.Select("ts")
for _, g := range query.GroupBy {
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
}
sb.SelectMore(spatial + " AS value")
sb.From("__temporal_aggregation_cte")
sb.GroupBy("ts")
sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)
q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args
}
func (b *MetricQueryStatementBuilder) buildTemporalAggDeltaFastPath(
start, end uint64,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
samplesTable string,
timeSeriesCTE string,
timeSeriesCTEArgs []any,
) (string, []any, error) {
@@ -242,8 +424,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggDeltaFastPath(
}
aggCol, err := AggregationColumnForSamplesTable(
start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality,
query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints,
samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation,
)
if err != nil {
return "", nil, err
@@ -260,8 +441,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggDeltaFastPath(
sb.SelectMore(fmt.Sprintf("%s AS value", aggCol))
tbl, _ := WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
sb.From(fmt.Sprintf("%s.%s AS points", DBName, tbl))
sb.From(fmt.Sprintf("%s.%s AS points", DBName, samplesTable))
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
sb.Where(
sb.In("metric_name", query.Aggregations[0].MetricName),
@@ -281,6 +461,7 @@ func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE(
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
tsTable string,
) (string, []any, error) {
sb := sqlbuilder.NewSelectBuilder()
@@ -304,8 +485,7 @@ func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE(
}
}
start, end, _, tbl := WhichTSTableToUse(start, end, query.Aggregations[0].TableHints)
sb.From(fmt.Sprintf("%s.%s", DBName, tbl))
sb.From(fmt.Sprintf("%s.%s", DBName, tsTable))
sb.Select("fingerprint")
for _, g := range query.GroupBy {
@@ -331,6 +511,12 @@ func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE(
sb.EQ("__normalized", false),
)
// the buffer holds both raw rows and the reduced catalog rows; the raw read
// only wants the original series
if tsTable == TimeseriesV4BufferLocalTableName {
sb.Where(sb.EQ("is_reduced", false))
}
if !preparedWhereClause.IsEmpty() {
sb.AddWhereClause(preparedWhereClause.WhereClause)
}
@@ -347,21 +533,23 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggregationCTE(
start, end uint64,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
_ map[string][]*telemetrytypes.TelemetryFieldKey,
samplesTable string,
timeSeriesCTE string,
timeSeriesCTEArgs []any,
) (string, []any, error) {
if query.Aggregations[0].Temporality == metrictypes.Delta {
return b.buildTemporalAggDelta(ctx, start, end, query, timeSeriesCTE, timeSeriesCTEArgs)
return b.buildTemporalAggDelta(ctx, start, end, query, samplesTable, timeSeriesCTE, timeSeriesCTEArgs)
} else if query.Aggregations[0].Temporality != metrictypes.Multiple {
return b.buildTemporalAggCumulativeOrUnspecified(ctx, start, end, query, timeSeriesCTE, timeSeriesCTEArgs)
return b.buildTemporalAggCumulativeOrUnspecified(ctx, start, end, query, samplesTable, timeSeriesCTE, timeSeriesCTEArgs)
}
return b.buildTemporalAggForMultipleTemporalities(ctx, start, end, query, timeSeriesCTE, timeSeriesCTEArgs)
return b.buildTemporalAggForMultipleTemporalities(ctx, start, end, query, samplesTable, timeSeriesCTE, timeSeriesCTEArgs)
}
func (b *MetricQueryStatementBuilder) buildTemporalAggDelta(
_ context.Context,
start, end uint64,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
samplesTable string,
timeSeriesCTE string,
timeSeriesCTEArgs []any,
) (string, []any, error) {
@@ -378,7 +566,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggDelta(
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
}
aggCol, err := AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
aggCol, err := AggregationColumnForSamplesTable(samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation)
if err != nil {
return "", nil, err
}
@@ -389,8 +577,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggDelta(
sb.SelectMore(fmt.Sprintf("%s AS per_series_value", aggCol))
tbl, _ := WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
sb.From(fmt.Sprintf("%s.%s AS points", DBName, tbl))
sb.From(fmt.Sprintf("%s.%s AS points", DBName, samplesTable))
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
sb.Where(
sb.In("metric_name", query.Aggregations[0].MetricName),
@@ -409,6 +596,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
_ context.Context,
start, end uint64,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
samplesTable string,
timeSeriesCTE string,
timeSeriesCTEArgs []any,
) (string, []any, error) {
@@ -424,14 +612,13 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
baseSb.SelectMore(fmt.Sprintf("`%s`", g.Name))
}
aggCol, err := AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
aggCol, err := AggregationColumnForSamplesTable(samplesTable, query.Aggregations[0].Temporality, query.Aggregations[0].TimeAggregation)
if err != nil {
return "", nil, err
}
baseSb.SelectMore(fmt.Sprintf("%s AS per_series_value", aggCol))
tbl, _ := WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
baseSb.From(fmt.Sprintf("%s.%s AS points", DBName, tbl))
baseSb.From(fmt.Sprintf("%s.%s AS points", DBName, samplesTable))
baseSb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
baseSb.Where(
baseSb.In("metric_name", query.Aggregations[0].MetricName),
@@ -475,6 +662,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggForMultipleTemporalities(
_ context.Context,
start, end uint64,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
samplesTable string,
timeSeriesCTE string,
timeSeriesCTEArgs []any,
) (string, []any, error) {
@@ -489,11 +677,11 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggForMultipleTemporalities(
sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
}
aggForDeltaTemporality, err := AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, metrictypes.Delta, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
aggForDeltaTemporality, err := AggregationColumnForSamplesTable(samplesTable, metrictypes.Delta, query.Aggregations[0].TimeAggregation)
if err != nil {
return "", nil, err
}
aggForCumulativeTemporality, err := AggregationColumnForSamplesTable(start, end, query.Aggregations[0].Type, metrictypes.Cumulative, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
aggForCumulativeTemporality, err := AggregationColumnForSamplesTable(samplesTable, metrictypes.Cumulative, query.Aggregations[0].TimeAggregation)
if err != nil {
return "", nil, err
}
@@ -521,8 +709,7 @@ func (b *MetricQueryStatementBuilder) buildTemporalAggForMultipleTemporalities(
sb.SelectMore(expr)
}
tbl, _ := WhichSamplesTableToUse(start, end, query.Aggregations[0].Type, query.Aggregations[0].TimeAggregation, query.Aggregations[0].TableHints)
sb.From(fmt.Sprintf("%s.%s AS points", DBName, tbl))
sb.From(fmt.Sprintf("%s.%s AS points", DBName, samplesTable))
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
sb.Where(
sb.In("metric_name", query.Aggregations[0].MetricName),

View File

@@ -30,6 +30,17 @@ const (
TimeseriesV41weekLocalTableName = "time_series_v4_1week"
AttributesMetadataTableName = "distributed_metadata"
AttributesMetadataLocalTableName = "metadata"
// The buffer holds raw points for ~24h; the reduced tables hold 60s
// aggregates of dropped-label series.
SamplesV4BufferTableName = "distributed_samples_v4_buffer"
SamplesV4BufferLocalTableName = "samples_v4_buffer"
TimeseriesV4BufferTableName = "distributed_time_series_v4_buffer"
TimeseriesV4BufferLocalTableName = "time_series_v4_buffer"
SamplesV4ReducedLastTableName = "distributed_samples_v4_reduced_last_60s"
SamplesV4ReducedSumTableName = "distributed_samples_v4_reduced_sum_60s"
TimeseriesV4ReducedTableName = "distributed_time_series_v4_reduced"
TimeseriesV4ReducedLocalTableName = "time_series_v4_reduced"
)
var (
@@ -49,8 +60,16 @@ var (
// in that order.
func WhichTSTableToUse(
start, end uint64,
useBuffer bool,
tableHints *metrictypes.MetricTableHints,
) (uint64, uint64, string, string) {
// the buffer holds the recent raw window for reduced metrics and has the same
// shape as time_series_v4; round the start to the hour like the v4 table.
if useBuffer {
start = start - (start % (oneHourInMilliseconds))
return start, end, TimeseriesV4BufferTableName, TimeseriesV4BufferLocalTableName
}
// if we have a hint for the table, we need to use it
// the hint will be used to override the default table selection logic
if tableHints != nil {
@@ -149,14 +168,20 @@ func WhichSamplesTableToUse(
start, end uint64,
metricType metrictypes.Type,
timeAggregation metrictypes.TimeAggregation,
useBuffer bool,
tableHints *metrictypes.MetricTableHints,
) (string, string) {
// the buffer holds the recent raw window for reduced metrics; same shape as samples_v4
if useBuffer {
return SamplesV4BufferTableName, SamplesV4BufferLocalTableName
}
// if we have a hint for the table, we need to use it
// the hint will be used to override the default table selection logic.
// SamplesTableName is the distributed name; derive the local via switch.
if tableHints != nil && tableHints.SamplesTableName != "" {
switch tableHints.SamplesTableName {
case SamplesV4TableName:
case SamplesV4TableName, SamplesV4BufferTableName:
return SamplesV4TableName, SamplesV4LocalTableName
case SamplesV4Agg5mTableName:
return SamplesV4Agg5mTableName, SamplesV4Agg5mLocalTableName
@@ -188,13 +213,10 @@ func WhichSamplesTableToUse(
}
func AggregationColumnForSamplesTable(
start, end uint64,
metricType metrictypes.Type,
tableName string,
temporality metrictypes.Temporality,
timeAggregation metrictypes.TimeAggregation,
tableHints *metrictypes.MetricTableHints,
) (string, error) {
tableName, _ := WhichSamplesTableToUse(start, end, metricType, timeAggregation, tableHints)
var aggregationColumn string
switch temporality {
case metrictypes.Delta:
@@ -202,7 +224,7 @@ func AggregationColumnForSamplesTable(
// although it doesn't make sense to use anyLast, avg, min, max, count on delta metrics,
// we are keeping it here to make sure that query will not be invalid
switch tableName {
case SamplesV4TableName:
case SamplesV4TableName, SamplesV4BufferTableName:
switch timeAggregation {
case metrictypes.TimeAggregationLatest:
aggregationColumn = "anyLast(value)"
@@ -244,7 +266,7 @@ func AggregationColumnForSamplesTable(
// for cumulative metrics, we only support `RATE`/`INCREASE`. The max value in window is
// used to calculate the sum which is then divided by the window size to get the rate
switch tableName {
case SamplesV4TableName:
case SamplesV4TableName, SamplesV4BufferTableName:
switch timeAggregation {
case metrictypes.TimeAggregationLatest:
aggregationColumn = "anyLast(value)"
@@ -284,7 +306,7 @@ func AggregationColumnForSamplesTable(
}
case metrictypes.Unspecified:
switch tableName {
case SamplesV4TableName:
case SamplesV4TableName, SamplesV4BufferTableName:
switch timeAggregation {
case metrictypes.TimeAggregationLatest:
aggregationColumn = "anyLast(value)"
@@ -332,6 +354,65 @@ func AggregationColumnForSamplesTable(
return aggregationColumn, nil
}
// WhichReducedSamplesTableToUse returns the 60s reduced samples table for a metric
// type: the last_60s table for gauge-like series, the sum_60s table for counters
// and histograms.
func WhichReducedSamplesTableToUse(metricType metrictypes.Type) string {
if metricType == metrictypes.SumType || metricType == metrictypes.HistogramType {
return SamplesV4ReducedSumTableName
}
return SamplesV4ReducedLastTableName
}
// ReducedValueColumn returns the reduced value column (and the avg-denominator
// weight) for a space aggregation. The reduced columns are pre-aggregated across
// the original series, so the space aggregation picks the underlying value; the
// sum table only has `sum`, so min/max across series have no column (ok=false).
func ReducedValueColumn(metricType metrictypes.Type, space metrictypes.SpaceAggregation) (value, weight string, ok bool) {
if metricType == metrictypes.SumType || metricType == metrictypes.HistogramType {
switch space {
case metrictypes.SpaceAggregationSum:
return "`sum`", "", true
case metrictypes.SpaceAggregationAvg:
return "`sum`", "`count_series`", true
}
return "", "", false
}
switch space {
case metrictypes.SpaceAggregationSum:
return "`sum_last`", "", true
case metrictypes.SpaceAggregationAvg:
return "`sum_last`", "`count_series`", true
case metrictypes.SpaceAggregationMin:
return "`min`", "", true
case metrictypes.SpaceAggregationMax:
return "`max`", "", true
}
return "", "", false
}
// ReducedTimeAggregationColumn applies the time aggregation to the reduced `value`
// column over the step's 60s buckets. latest uses argMax over the bucket timestamp
// (the buckets have no read order); rate divides the per-step sum by the step.
func ReducedTimeAggregationColumn(timeAggregation metrictypes.TimeAggregation, stepSec int64) string {
switch timeAggregation {
case metrictypes.TimeAggregationLatest:
return "argMax(value, unix_milli)"
case metrictypes.TimeAggregationAvg:
return "avg(value)"
case metrictypes.TimeAggregationMin:
return "min(value)"
case metrictypes.TimeAggregationMax:
return "max(value)"
case metrictypes.TimeAggregationCount:
return "count(value)"
case metrictypes.TimeAggregationRate:
return fmt.Sprintf("sum(value) / %d", stepSec)
default: // sum, increase
return "sum(value)"
}
}
func AggregationQueryForHistogramCountWithParams(param *metrictypes.ComparisonSpaceAggregationParam) (string, error) {
if param == nil {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "no aggregation param provided for histogram count")

View File

@@ -38,7 +38,7 @@ func newTestDashboardV2(t *testing.T, orgID valuer.UUID, source Source) *Dashboa
FillMode: FillModeSolid,
SpanGaps: SpanGaps{FillLessThan: valuer.MustParseTextDuration("60s")},
},
Legend: Legend{Position: LegendPositionBottom, Mode: LegendModeList},
Legend: Legend{Position: LegendPositionBottom},
},
},
Queries: []Query{

View File

@@ -48,42 +48,7 @@ func (d *DashboardSpec) UnmarshalJSON(data []byte) error {
// ══════════════════════════════════════════════
func (d *DashboardSpec) Validate() error {
if err := d.validateVariables(); err != nil {
return err
}
if err := d.validatePanels(); err != nil {
return err
}
return d.validateLayouts()
}
// validateVariables rejects two variables sharing the same name.
func (d *DashboardSpec) validateVariables() error {
seen := make(map[string]struct{}, len(d.Variables))
for i, v := range d.Variables {
var name string
switch s := v.Spec.(type) {
case *ListVariableSpec:
name = s.Name
case *TextVariableSpec:
name = s.Name
default:
// Unreachable via UnmarshalJSON; reaching here means a Go caller broke the Kind/Spec pairing.
return errors.NewInternalf(errors.CodeInternal, "spec.variables[%d].spec: unexpected variable spec type %T", i, v.Spec)
}
if _, dup := seen[name]; dup {
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "spec.variables[%d]: duplicate variable name %q", i, name)
}
seen[name] = struct{}{}
}
return nil
}
func (d *DashboardSpec) validatePanels() error {
for key, panel := range d.Panels {
if err := common.ValidateID(key); err != nil {
return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "spec.panels: %s", err.Error())
}
if panel == nil {
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "spec.panels.%s: panel must not be null", key)
}
@@ -104,13 +69,6 @@ func (d *DashboardSpec) validatePanels() error {
}
func validateQueryAllowedForPanel(plugin QueryPlugin, allowed []QueryPluginKind, panelKind PanelPluginKind, path string) error {
compositeSubQueryTypeToPluginKind := map[qb.QueryType]QueryPluginKind{
qb.QueryTypeBuilder: QueryKindBuilder,
qb.QueryTypeFormula: QueryKindFormula,
qb.QueryTypeTraceOperator: QueryKindTraceOperator,
qb.QueryTypePromQL: QueryKindPromQL,
qb.QueryTypeClickHouseSQL: QueryKindClickHouseSQL,
}
if !slices.Contains(allowed, plugin.Kind) {
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput,
"%s: query kind %q is not supported by panel kind %q", path, plugin.Kind, panelKind)
@@ -138,35 +96,12 @@ func validateQueryAllowedForPanel(plugin QueryPlugin, allowed []QueryPluginKind,
return nil
}
// validateLayouts rejects grid items referencing a panel that doesn't exist.
func (d *DashboardSpec) validateLayouts() error {
for li, layout := range d.Layouts {
grid, ok := layout.Spec.(*dashboard.GridLayoutSpec)
if !ok {
// Unreachable via UnmarshalJSON; reaching here means a Go caller broke the Kind/Spec pairing.
return errors.NewInternalf(errors.CodeInternal, "spec.layouts[%d].spec: unexpected layout spec type %T", li, layout.Spec)
}
for ii, item := range grid.Items {
path := fmt.Sprintf("spec.layouts[%d].spec.items[%d].content", li, ii)
if item.Content == nil {
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "%s: content reference is required", path)
}
key, err := panelKeyFromRef(item.Content.Path, item.Content.Ref, path)
if err != nil {
return err
}
if _, ok := d.Panels[key]; !ok {
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "%s: references unknown panel %q", path, key)
}
}
var (
compositeSubQueryTypeToPluginKind = map[qb.QueryType]QueryPluginKind{
qb.QueryTypeBuilder: QueryKindBuilder,
qb.QueryTypeFormula: QueryKindFormula,
qb.QueryTypeTraceOperator: QueryKindTraceOperator,
qb.QueryTypePromQL: QueryKindPromQL,
qb.QueryTypeClickHouseSQL: QueryKindClickHouseSQL,
}
return nil
}
// panelKeyFromRef extracts <key> from a "#/spec/panels/<key>" content ref.
func panelKeyFromRef(refPath []string, ref string, path string) (string, error) {
if len(refPath) != 3 || refPath[0] != "spec" || refPath[1] != "panels" {
return "", errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "%s: %q must reference a panel as \"#/spec/panels/<key>\"", path, ref)
}
return refPath[2], nil
}
)

View File

@@ -73,7 +73,7 @@ func (p PatchableDashboardV2) Apply(existing *DashboardV2) (*UpdatableDashboardV
}
patched, err := p.patch.ApplyWithOptions(raw, &jsonpatch.ApplyOptions{AllowMissingPathOnRemove: true, EnsurePathExistsOnAdd: true})
if err != nil {
return nil, errors.Wrap(err, errors.TypeInvalidInput, ErrCodeDashboardInvalidPatch, "JSON Patch could not be applied to the target dashboard").WithAdditional(err.Error())
return nil, errors.Wrap(err, errors.TypeInvalidInput, ErrCodeDashboardInvalidPatch, "JSON Patch could not be applied to the target dashboard")
}
out := &UpdatableDashboardV2{}
if err := json.Unmarshal(patched, out); err != nil {

View File

@@ -405,7 +405,6 @@ func TestPatchableDashboardV2_Apply(t *testing.T) {
out, err := decode(t, `[
{"op": "replace", "path": "/spec/display/name", "value": "Multi-step"},
{"op": "remove", "path": "/spec/panels/p2"},
{"op": "remove", "path": "/spec/layouts/0/spec/items/1"},
{"op": "add", "path": "/tags/-", "value": {"key": "env", "value": "staging"}}
]`).Apply(base)
require.NoError(t, err)

View File

@@ -112,174 +112,6 @@ func TestValidateOnlyVariables(t *testing.T) {
require.NoError(t, err, "expected valid")
}
func TestInvalidateDuplicateVariableNames(t *testing.T) {
data := []byte(`{
"variables": [
{
"kind": "TextVariable",
"spec": {"name": "env", "value": "prod"}
},
{
"kind": "ListVariable",
"spec": {
"name": "env",
"allowAllValue": false,
"allowMultiple": false,
"plugin": {
"kind": "signoz/DynamicVariable",
"spec": {"name": "service.name", "signal": "metrics"}
}
}
}
],
"layouts": []
}`)
_, err := unmarshalDashboard(data)
require.Error(t, err, "expected error for duplicate variable name")
require.Contains(t, err.Error(), `duplicate variable name "env"`)
}
func TestInvalidateVariableNameWithInvalidChars(t *testing.T) {
listVarWithName := func(name string) []byte {
return []byte(`{
"variables": [
{
"kind": "ListVariable",
"spec": {
"name": "` + name + `",
"allowAllValue": false,
"allowMultiple": false,
"plugin": {
"kind": "signoz/DynamicVariable",
"spec": {"name": "service.name", "signal": "metrics"}
}
}
}
],
"layouts": []
}`)
}
for _, name := range []string{"my var", "cost$", "bad!", "a/b"} {
t.Run(name, func(t *testing.T) {
_, err := unmarshalDashboard(listVarWithName(name))
require.Error(t, err, "expected error for invalid variable name %q", name)
require.Contains(t, err.Error(), "is not a correct name")
})
}
for _, name := range []string{"service", "my_var", "MY_VAR", "MixedCase9", "with-hyphen", "with.dot"} {
t.Run(name, func(t *testing.T) {
_, err := unmarshalDashboard(listVarWithName(name))
require.NoError(t, err, "expected valid variable name %q", name)
})
}
t.Run("digits only", func(t *testing.T) {
_, err := unmarshalDashboard(listVarWithName("123"))
require.Error(t, err)
require.Contains(t, err.Error(), "cannot contain only digits")
})
}
func TestInvalidatePanelKey(t *testing.T) {
data := []byte(`{
"panels": {
"bad key!": {
"kind": "Panel",
"spec": {
"plugin": {"kind": "signoz/TablePanel", "spec": {}},
"queries": [{
"kind": "time_series",
"spec": {"plugin": {"kind": "signoz/BuilderQuery", "spec": {
"name": "A", "signal": "logs", "aggregations": [{"expression": "count()"}]
}}}
}]
}
}
},
"layouts": []
}`)
_, err := unmarshalDashboard(data)
require.Error(t, err, "expected error for invalid panel key")
require.Contains(t, err.Error(), "is not a correct name")
}
func TestInvalidateListVariableCrossFields(t *testing.T) {
listVar := func(specFields string) []byte {
return []byte(`{
"variables": [
{
"kind": "ListVariable",
"spec": {
"name": "service",
` + specFields + `
"plugin": {
"kind": "signoz/DynamicVariable",
"spec": {"name": "service.name", "signal": "metrics"}
}
}
}
],
"layouts": []
}`)
}
t.Run("customAllValue without allowAllValue", func(t *testing.T) {
_, err := unmarshalDashboard(listVar(`"allowAllValue": false, "allowMultiple": false, "customAllValue": "*",`))
require.Error(t, err)
require.Contains(t, err.Error(), "customAllValue cannot be set")
})
t.Run("list defaultValue without allowMultiple", func(t *testing.T) {
_, err := unmarshalDashboard(listVar(`"allowAllValue": false, "allowMultiple": false, "defaultValue": ["a", "b"],`))
require.Error(t, err)
require.Contains(t, err.Error(), "allowMultiple")
})
t.Run("single-element list default without allowMultiple", func(t *testing.T) {
_, err := unmarshalDashboard(listVar(`"allowAllValue": false, "allowMultiple": false, "defaultValue": ["only"],`))
require.Error(t, err)
require.Contains(t, err.Error(), "allowMultiple")
})
t.Run("valid sort is accepted", func(t *testing.T) {
_, err := unmarshalDashboard(listVar(`"sort": "alphabetical-asc",`))
require.NoError(t, err)
})
t.Run("unknown sort is rejected", func(t *testing.T) {
_, err := unmarshalDashboard(listVar(`"sort": "bogus",`))
require.Error(t, err)
require.Contains(t, err.Error(), "unknown sort")
})
}
func TestInvalidateEmptyVariableName(t *testing.T) {
cases := map[string][]byte{
"text variable": []byte(`{
"variables": [{"kind": "TextVariable", "spec": {"name": "", "value": "x"}}],
"layouts": []
}`),
"list variable": []byte(`{
"variables": [{
"kind": "ListVariable",
"spec": {
"name": "",
"allowAllValue": false,
"allowMultiple": false,
"plugin": {"kind": "signoz/DynamicVariable", "spec": {"name": "service.name", "signal": "metrics"}}
}
}],
"layouts": []
}`),
}
for name, data := range cases {
t.Run(name, func(t *testing.T) {
_, err := unmarshalDashboard(data)
require.Error(t, err, "expected error for empty variable name")
require.Contains(t, err.Error(), "name cannot be empty")
})
}
}
func TestInvalidateUnknownPluginKind(t *testing.T) {
tests := []struct {
name string
@@ -438,65 +270,6 @@ func TestInvalidateOneInvalidPanel(t *testing.T) {
require.Contains(t, err.Error(), "FakePanel", "error should mention FakePanel")
}
func TestInvalidateLayoutPanelReferences(t *testing.T) {
validPanels := `"panels": {
"p1": {
"kind": "Panel",
"spec": {
"plugin": {"kind": "signoz/TablePanel", "spec": {}},
"queries": [{
"kind": "time_series",
"spec": {"plugin": {"kind": "signoz/BuilderQuery", "spec": {
"name": "A", "signal": "logs", "aggregations": [{"expression": "count()"}]
}}}
}]
}
}
}`
layout := func(items string) []byte {
return []byte(`{` + validPanels + `, "layouts": [{"kind": "Grid", "spec": {"items": [` + items + `]}}]}`)
}
tests := []struct {
name string
data []byte
wantContain string
}{
{
name: "reference to unknown panel",
data: layout(`{"x": 0, "y": 0, "width": 6, "height": 6, "content": {"$ref": "#/spec/panels/ghost"}}`),
wantContain: `references unknown panel "ghost"`,
},
{
name: "reference not pointing at a panel",
data: layout(`{"x": 0, "y": 0, "width": 6, "height": 6, "content": {"$ref": "#/spec/variables/p1"}}`),
wantContain: "must reference a panel",
},
{
name: "reference missing spec prefix",
data: layout(`{"x": 0, "y": 0, "width": 6, "height": 6, "content": {"$ref": "#/panels/p1"}}`),
wantContain: "must reference a panel",
},
{
name: "valid reference",
data: layout(`{"x": 0, "y": 0, "width": 6, "height": 6, "content": {"$ref": "#/spec/panels/p1"}}`),
wantContain: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := unmarshalDashboard(tt.data)
if tt.wantContain == "" {
require.NoError(t, err)
return
}
require.Error(t, err)
require.Contains(t, err.Error(), tt.wantContain)
})
}
}
func TestRejectUnknownFieldsInPluginSpec(t *testing.T) {
tests := []struct {
name string
@@ -796,24 +569,6 @@ func TestInvalidateBadPanelSpecValues(t *testing.T) {
}`,
wantContain: "legend position",
},
{
name: "bad legend mode",
data: `{
"panels": {
"p1": {
"kind": "Panel",
"spec": {
"plugin": {
"kind": "signoz/BarChartPanel",
"spec": {"legend": {"mode": "grid"}}
}
}
}
},
"layouts": []
}`,
wantContain: "legend mode",
},
{
name: "bad threshold format",
data: `{
@@ -879,39 +634,6 @@ func TestInvalidateBadPanelSpecValues(t *testing.T) {
}
}
// Label on ThresholdWithLabel is optional — the backend never reads it, so a
// threshold with an omitted or empty label must validate cleanly.
func TestThresholdLabelOptional(t *testing.T) {
for _, tt := range []struct {
name string
threshold string
}{
{name: "label omitted", threshold: `{"value": 100, "color": "Red"}`},
{name: "label empty", threshold: `{"value": 100, "color": "Red", "label": ""}`},
} {
t.Run(tt.name, func(t *testing.T) {
data := []byte(`{
"panels": {
"p1": {
"kind": "Panel",
"spec": {
"plugin": {"kind": "signoz/TimeSeriesPanel", "spec": {"thresholds": [` + tt.threshold + `]}},
"queries": [{"kind": "time_series", "spec": {"plugin": {"kind": "signoz/PromQLQuery", "spec": {"name": "A", "query": "up"}}}}]
}
}
},
"layouts": []
}`)
d, err := unmarshalDashboard(data)
require.NoError(t, err, "threshold without a label should validate")
spec := d.Panels["p1"].Spec.Plugin.Spec.(*TimeSeriesPanelSpec)
require.Len(t, spec.Thresholds, 1)
require.Empty(t, spec.Thresholds[0].Label, "label should remain empty")
})
}
}
func TestInvalidatePanelWithoutQueries(t *testing.T) {
data := []byte(`{
"panels": {
@@ -1027,6 +749,11 @@ func TestValidateRequiredFields(t *testing.T) {
data: wrapPanel("signoz/TimeSeriesPanel", `{"thresholds": [{"value": 100, "label": "high", "color": ""}]}`),
wantContain: "Color",
},
{
name: "ThresholdWithLabel missing label",
data: wrapPanel("signoz/TimeSeriesPanel", `{"thresholds": [{"value": 100, "color": "Red", "label": ""}]}`),
wantContain: "Label",
},
{
name: "ComparisonThreshold missing value",
data: wrapPanel("signoz/NumberPanel", `{"thresholds": [{"operator": "above", "format": "text", "color": "Red"}]}`),
@@ -1084,11 +811,10 @@ func TestTimeSeriesPanelDefaults(t *testing.T) {
require.Equal(t, "2", spec.Formatting.DecimalPrecision.ValueOrDefault(), "expected DecimalPrecision default 2")
require.Equal(t, "spline", spec.ChartAppearance.LineInterpolation.ValueOrDefault(), "expected LineInterpolation default spline")
require.Equal(t, "solid", spec.ChartAppearance.LineStyle.ValueOrDefault(), "expected LineStyle default solid")
require.Equal(t, "none", spec.ChartAppearance.FillMode.ValueOrDefault(), "expected FillMode default none")
require.Equal(t, "solid", spec.ChartAppearance.FillMode.ValueOrDefault(), "expected FillMode default solid")
require.False(t, spec.ChartAppearance.SpanGaps.FillOnlyBelow, "expected SpanGaps.FillOnlyBelow default false")
require.Equal(t, "global_time", spec.Visualization.TimePreference.ValueOrDefault(), "expected TimePreference default global_time")
require.Equal(t, "bottom", spec.Legend.Position.ValueOrDefault(), "expected LegendPosition default bottom")
require.Equal(t, "list", spec.Legend.Mode.ValueOrDefault(), "expected LegendMode default list")
// Re-marshal the full dashboard (what we'd store in DB / return in API response)
// and verify the output contains the default values.
@@ -1099,10 +825,9 @@ func TestTimeSeriesPanelDefaults(t *testing.T) {
"decimalPrecision": `"2"`,
"lineInterpolation": `"spline"`,
"lineStyle": `"solid"`,
"fillMode": `"none"`,
"fillMode": `"solid"`,
"timePreference": `"global_time"`,
"position": `"bottom"`,
"mode": `"list"`,
} {
assert.Contains(t, outputStr, `"`+field+`":`+want, "expected stored/response JSON to contain %s:%s", field, want)
}
@@ -1205,7 +930,7 @@ func TestStorageRoundTrip(t *testing.T) {
assert.Equal(t, "2", tsSpec.Formatting.DecimalPrecision.ValueOrDefault())
assert.Equal(t, "spline", tsSpec.ChartAppearance.LineInterpolation.ValueOrDefault())
assert.Equal(t, "solid", tsSpec.ChartAppearance.LineStyle.ValueOrDefault())
assert.Equal(t, "none", tsSpec.ChartAppearance.FillMode.ValueOrDefault())
assert.Equal(t, "solid", tsSpec.ChartAppearance.FillMode.ValueOrDefault())
assert.Equal(t, "global_time", tsSpec.Visualization.TimePreference.ValueOrDefault())
assert.Equal(t, "bottom", tsSpec.Legend.Position.ValueOrDefault())
numSpec := d.Panels["p2"].Spec.Plugin.Spec.(*NumberPanelSpec)
@@ -1225,7 +950,7 @@ func TestStorageRoundTrip(t *testing.T) {
assert.Equal(t, "2", tsLoaded.Formatting.DecimalPrecision.ValueOrDefault(), "after load")
assert.Equal(t, "spline", tsLoaded.ChartAppearance.LineInterpolation.ValueOrDefault(), "after load")
assert.Equal(t, "solid", tsLoaded.ChartAppearance.LineStyle.ValueOrDefault(), "after load")
assert.Equal(t, "none", tsLoaded.ChartAppearance.FillMode.ValueOrDefault(), "after load")
assert.Equal(t, "solid", tsLoaded.ChartAppearance.FillMode.ValueOrDefault(), "after load")
assert.Equal(t, "global_time", tsLoaded.Visualization.TimePreference.ValueOrDefault(), "after load")
assert.Equal(t, "bottom", tsLoaded.Legend.Position.ValueOrDefault(), "after load")
numLoaded := loaded.Panels["p2"].Spec.Plugin.Spec.(*NumberPanelSpec)
@@ -1241,7 +966,7 @@ func TestStorageRoundTrip(t *testing.T) {
"decimalPrecision": `"2"`,
"lineInterpolation": `"spline"`,
"lineStyle": `"solid"`,
"fillMode": `"none"`,
"fillMode": `"solid"`,
"timePreference": `"global_time"`,
"position": `"bottom"`,
"format": `"text"`,

View File

@@ -30,7 +30,6 @@ func TestDashboardSpecMatchesPerses(t *testing.T) {
{"DatasourceSpec", typeOf[DatasourceSpec](), typeOf[datasource.Spec]()},
{"Variable", typeOf[Variable](), typeOf[dashboard.Variable]()},
{"ListVariableSpec", typeOf[ListVariableSpec](), typeOf[dashboard.ListVariableSpec]()},
{"TextVariableSpec", typeOf[TextVariableSpec](), typeOf[dashboard.TextVariableSpec]()},
{"Layout", typeOf[Layout](), typeOf[dashboard.Layout]()},
}

View File

@@ -51,7 +51,7 @@ func (p *PanelPlugin) UnmarshalJSON(data []byte) error {
return err
}
p.Kind = PanelPluginKind(kind)
p.Spec = *spec
p.Spec = spec
return nil
}
@@ -110,7 +110,7 @@ func (p *QueryPlugin) UnmarshalJSON(data []byte) error {
return err
}
p.Kind = QueryPluginKind(kind)
p.Spec = *spec
p.Spec = spec
return nil
}
@@ -165,7 +165,7 @@ func (p *VariablePlugin) UnmarshalJSON(data []byte) error {
return err
}
p.Kind = VariablePluginKind(kind)
p.Spec = *spec
p.Spec = spec
return nil
}
@@ -215,7 +215,7 @@ func (p *DatasourcePlugin) UnmarshalJSON(data []byte) error {
return err
}
p.Kind = DatasourcePluginKind(kind)
p.Spec = *spec
p.Spec = spec
return nil
}
@@ -297,7 +297,8 @@ func extractKindAndSpec(data []byte) (string, []byte, error) {
return head.Kind, head.Spec, nil
}
func decodeSpec[T any](specJSON []byte, target T, kind string) (*T, error) {
// decodeSpec strict-decodes a spec JSON into target and runs struct-tag validation (go-playground/validator).
func decodeSpec(specJSON []byte, target any, kind string) (any, error) {
if len(specJSON) == 0 {
return nil, errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "kind %q: spec is required", kind)
}
@@ -309,12 +310,7 @@ func decodeSpec[T any](specJSON []byte, target T, kind string) (*T, error) {
if err := validator.New().Struct(target); err != nil {
return nil, errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "kind %q: spec failed validation", kind)
}
if v, ok := any(target).(interface{ validate() error }); ok {
if err := v.validate(); err != nil {
return nil, errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "kind %q: %s", kind, err.Error())
}
}
return &target, nil
return target, nil
}
// signozDiscriminatorKey is the extension key that signoz.attachDiscriminators

View File

@@ -4,11 +4,9 @@ import (
"encoding/json"
"maps"
"slices"
"strconv"
"github.com/SigNoz/signoz/pkg/errors"
qb "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/perses/spec/go/common"
"github.com/perses/spec/go/dashboard"
"github.com/perses/spec/go/dashboard/variable"
@@ -86,7 +84,7 @@ type QuerySpec struct {
// ══════════════════════════════════════════════
// Variable is the list/text sum type. Spec is set to *ListVariableSpec or
// *TextVariableSpec by UnmarshalJSON based on Kind. The schema is a
// *dashboard.TextVariableSpec by UnmarshalJSON based on Kind. The schema is a
// discriminated oneOf (see JSONSchemaOneOf).
type Variable struct {
Kind variable.Kind `json:"kind" required:"true"`
@@ -96,7 +94,7 @@ type Variable struct {
func (Variable) PrepareJSONSchema(s *jsonschema.Schema) error {
return markDiscriminator(s, "kind", map[string]string{
string(variable.KindList): schemaRef("DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDashboardtypesListVariableSpec"),
string(variable.KindText): schemaRef("DashboardtypesVariableEnvelopeGithubComSigNozSignozPkgTypesDashboardtypesTextVariableSpec"),
string(variable.KindText): schemaRef("DashboardtypesVariableEnvelopeGithubComPersesSpecGoDashboardTextVariableSpec"),
})
}
@@ -112,14 +110,14 @@ func (v *Variable) UnmarshalJSON(data []byte) error {
return err
}
v.Kind = variable.KindList
v.Spec = *spec
v.Spec = spec
case string(variable.KindText):
spec, err := decodeSpec(specJSON, new(TextVariableSpec), kind)
spec, err := decodeSpec(specJSON, new(dashboard.TextVariableSpec), kind)
if err != nil {
return err
}
v.Kind = variable.KindText
v.Spec = *spec
v.Spec = spec
default:
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "unknown variable kind %q; allowed values: %s", kind, allowedValuesForKind([]variable.Kind{variable.KindList, variable.KindText}))
}
@@ -129,7 +127,7 @@ func (v *Variable) UnmarshalJSON(data []byte) error {
func (Variable) JSONSchemaOneOf() []any {
return []any{
VariableEnvelope[ListVariableSpec]{Kind: string(variable.KindList)},
VariableEnvelope[TextVariableSpec]{Kind: string(variable.KindText)},
VariableEnvelope[dashboard.TextVariableSpec]{Kind: string(variable.KindText)},
}
}
@@ -145,106 +143,15 @@ func (v VariableEnvelope[S]) PrepareJSONSchema(s *jsonschema.Schema) error {
// ListVariableSpec mirrors dashboard.ListVariableSpec (variable.ListSpec
// fields + Name) but with a typed VariablePlugin replacing common.Plugin.
type ListVariableSpec struct {
Display *Display `json:"display,omitempty"`
Display Display `json:"display" required:"true"`
DefaultValue *variable.DefaultValue `json:"defaultValue,omitempty"`
AllowAllValue bool `json:"allowAllValue"`
AllowMultiple bool `json:"allowMultiple"`
CustomAllValue string `json:"customAllValue,omitempty"`
CapturingRegexp string `json:"capturingRegexp,omitempty"`
Sort ListVariableSpecSort `json:"sort,omitzero"`
Sort *variable.Sort `json:"sort,omitempty"`
Plugin VariablePlugin `json:"plugin"`
Name string `json:"name" required:"true" minLength:"1"`
}
// validate mirrors perses ListVariableSpec validation (plus the digits-only name
// check perses only applies to text variables); run by decodeSpec on unmarshal.
func (s *ListVariableSpec) validate() error {
if err := common.ValidateID(s.Name); err != nil {
return err
}
if _, err := strconv.Atoi(s.Name); err == nil {
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "variable name cannot contain only digits")
}
if s.CustomAllValue != "" && !s.AllowAllValue {
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "customAllValue cannot be set if allowAllValue is not set to true")
}
if s.DefaultValue != nil && len(s.DefaultValue.SliceValues) > 0 && !s.AllowMultiple {
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "defaultValue cannot be a list if allowMultiple is not set to true")
}
return nil
}
// ListVariableSpecSort is the value-list sort method, mirrored from Perses as a
// stable enum so the allowed values surface in the generated OpenAPI schema.
type ListVariableSpecSort struct{ valuer.String }
var (
SortNone = ListVariableSpecSort{valuer.NewString("none")}
SortAlphabeticalAsc = ListVariableSpecSort{valuer.NewString("alphabetical-asc")}
SortAlphabeticalDesc = ListVariableSpecSort{valuer.NewString("alphabetical-desc")}
SortNumericalAsc = ListVariableSpecSort{valuer.NewString("numerical-asc")}
SortNumericalDesc = ListVariableSpecSort{valuer.NewString("numerical-desc")}
SortAlphabeticalCaseInsensitiveAsc = ListVariableSpecSort{valuer.NewString("alphabetical-ci-asc")}
SortAlphabeticalCaseInsensitiveDesc = ListVariableSpecSort{valuer.NewString("alphabetical-ci-desc")}
)
func (ListVariableSpecSort) Enum() []any {
return []any{
SortNone,
SortAlphabeticalAsc,
SortAlphabeticalDesc,
SortNumericalAsc,
SortNumericalDesc,
SortAlphabeticalCaseInsensitiveAsc,
SortAlphabeticalCaseInsensitiveDesc,
}
}
func (s ListVariableSpecSort) IsValid() bool {
return slices.ContainsFunc(s.Enum(), func(v any) bool { return v == s })
}
// UnmarshalJSON validates against the enum on decode (valuer.String alone
// accepts any string). An empty value is allowed and means "no sort", matching
// Perses.
func (s *ListVariableSpecSort) UnmarshalJSON(data []byte) error {
var v string
if err := json.Unmarshal(data, &v); err != nil {
return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid sort: must be a string, one of `none`, `alphabetical-asc`, `alphabetical-desc`, `numerical-asc`, `numerical-desc`, `alphabetical-ci-asc`, or `alphabetical-ci-desc`")
}
if v == "" {
*s = ListVariableSpecSort{}
return nil
}
sort := ListVariableSpecSort{valuer.NewString(v)}
if !sort.IsValid() {
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "unknown sort %q: must be `none`, `alphabetical-asc`, `alphabetical-desc`, `numerical-asc`, `numerical-desc`, `alphabetical-ci-asc`, or `alphabetical-ci-desc`", v)
}
*s = sort
return nil
}
// TextVariableSpec replicates dashboard.TextVariableSpec so name can carry the
// required/non-empty schema tags perses leaves off.
type TextVariableSpec struct {
Display *Display `json:"display,omitempty"`
Value string `json:"value"`
Constant bool `json:"constant,omitempty"`
Name string `json:"name" required:"true" minLength:"1"`
}
// validate mirrors perses TextVariableSpec validation; run by decodeSpec on unmarshal.
func (s *TextVariableSpec) validate() error {
if err := common.ValidateID(s.Name); err != nil {
return err
}
if _, err := strconv.Atoi(s.Name); err == nil {
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "variable name cannot contain only digits")
}
if s.Value == "" && s.Constant {
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "value for a constant text variable cannot be empty")
}
return nil
Name string `json:"name"`
}
// ══════════════════════════════════════════════
@@ -287,7 +194,7 @@ func (l *Layout) UnmarshalJSON(data []byte) error {
return err
}
l.Kind = dashboard.LayoutKind(kind)
l.Spec = *spec
l.Spec = spec
return nil
}

View File

@@ -241,7 +241,6 @@ type TableFormatting struct {
type Legend struct {
Position LegendPosition `json:"position"`
Mode LegendMode `json:"mode"`
CustomColors map[string]string `json:"customColors"`
}
@@ -249,7 +248,7 @@ type ThresholdWithLabel struct {
Value float64 `json:"value" validate:"required" required:"true"`
Unit string `json:"unit"`
Color string `json:"color" validate:"required" required:"true"`
Label string `json:"label"`
Label string `json:"label" validate:"required" required:"true"`
}
type ComparisonThreshold struct {
@@ -359,47 +358,6 @@ func (l *LegendPosition) UnmarshalJSON(data []byte) error {
}
}
type LegendMode struct{ valuer.String }
var (
LegendModeList = LegendMode{valuer.NewString("list")} // default
LegendModeTable = LegendMode{valuer.NewString("table")}
)
func (LegendMode) Enum() []any {
return []any{LegendModeList} // others are not supported in UI yet
}
func (m LegendMode) ValueOrDefault() string {
if m.IsZero() {
return LegendModeList.StringValue()
}
return m.StringValue()
}
func (m LegendMode) MarshalJSON() ([]byte, error) {
return json.Marshal(m.ValueOrDefault())
}
func (m *LegendMode) UnmarshalJSON(data []byte) error {
var v string
if err := json.Unmarshal(data, &v); err != nil {
return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid legend mode: must be a string, one of `list` or `table`")
}
if v == "" {
*m = LegendModeList
return nil
}
lm := LegendMode{valuer.NewString(v)}
switch lm {
case LegendModeList, LegendModeTable:
*m = lm
return nil
default:
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "invalid legend mode %q: must be `list` or `table`", v)
}
}
type ThresholdFormat struct{ valuer.String }
var (
@@ -576,9 +534,9 @@ func (ls *LineStyle) UnmarshalJSON(data []byte) error {
type FillMode struct{ valuer.String }
var (
FillModeSolid = FillMode{valuer.NewString("solid")}
FillModeSolid = FillMode{valuer.NewString("solid")} // default
FillModeGradient = FillMode{valuer.NewString("gradient")}
FillModeNone = FillMode{valuer.NewString("none")} // default
FillModeNone = FillMode{valuer.NewString("none")}
)
func (FillMode) Enum() []any {
@@ -587,7 +545,7 @@ func (FillMode) Enum() []any {
func (fm FillMode) ValueOrDefault() string {
if fm.IsZero() {
return FillModeNone.StringValue()
return FillModeSolid.StringValue()
}
return fm.StringValue()
}
@@ -602,7 +560,7 @@ func (fm *FillMode) UnmarshalJSON(data []byte) error {
return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid fill mode: must be a string, one of `solid`, `gradient`, or `none`")
}
if v == "" {
*fm = FillModeNone
*fm = FillModeSolid
return nil
}
val := FillMode{valuer.NewString(v)}
@@ -615,9 +573,12 @@ func (fm *FillMode) UnmarshalJSON(data []byte) error {
}
}
// SpanGaps controls whether lines connect across null values.
// When FillOnlyBelow is false (default), all gaps are connected.
// When FillOnlyBelow is true, only gaps smaller than FillLessThan are connected.
type SpanGaps struct {
FillOnlyBelow bool `json:"fillOnlyBelow" description:"Controls whether lines connect across null values. When false (default), all gaps are connected. When true, only gaps smaller than fillLessThan are connected."`
FillLessThan valuer.TextDuration `json:"fillLessThan" description:"The maximum gap size to connect when fillOnlyBelow is true. Gaps larger than this duration are left disconnected."`
FillOnlyBelow bool `json:"fillOnlyBelow"`
FillLessThan valuer.TextDuration `json:"fillLessThan"`
}
type PrecisionOption struct{ valuer.String }

View File

@@ -480,7 +480,9 @@ type MetricAggregation struct {
// value filter to apply to the query
ValueFilter *metrictypes.MetricValueFilter `json:"-"`
// reduce to operator for metric scalar requests
ReduceTo ReduceTo `json:"reduceTo,omitempty"`
ReduceTo ReduceTo `json:"reduceTo,omitzero"`
Reduced bool `json:"-"`
}
// Copy creates a deep copy of MetricAggregation.

View File

@@ -4,6 +4,7 @@ import (
"context"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
"github.com/SigNoz/signoz/pkg/valuer"
)
// MetadataStore is the interface for the telemetry metadata store.
@@ -26,12 +27,12 @@ type MetadataStore interface {
GetAllValues(ctx context.Context, fieldValueSelector *FieldValueSelector) (*TelemetryFieldValues, bool, error)
// FetchTemporality fetches the temporality for metric
FetchTemporality(ctx context.Context, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricName string) (metrictypes.Temporality, error)
FetchTemporality(ctx context.Context, orgID valuer.UUID, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricName string) (metrictypes.Temporality, error)
// FetchTemporalityMulti fetches the temporality for multiple metrics
FetchTemporalityMulti(ctx context.Context, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricNames ...string) (map[string]metrictypes.Temporality, error)
FetchTemporalityMulti(ctx context.Context, orgID valuer.UUID, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricNames ...string) (map[string]metrictypes.Temporality, error)
FetchTemporalityAndTypeMulti(ctx context.Context, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricNames ...string) (map[string]metrictypes.Temporality, map[string]metrictypes.Type, error)
FetchTemporalityAndTypeMulti(ctx context.Context, orgID valuer.UUID, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricNames ...string) (map[string]metrictypes.Temporality, map[string]metrictypes.Type, map[string]bool, error)
// ListLogsJSONIndexes lists the JSON indexes for the logs table.
ListLogsJSONIndexes(ctx context.Context, filters ...string) ([]TelemetryFieldKeySkipIndex, error)

View File

@@ -6,6 +6,7 @@ import (
"github.com/SigNoz/signoz/pkg/types/metrictypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
)
// MockMetadataStore implements the MetadataStore interface for testing purposes.
@@ -16,6 +17,7 @@ type MockMetadataStore struct {
AllValuesMap map[string]*telemetrytypes.TelemetryFieldValues
TemporalityMap map[string]metrictypes.Temporality
TypeMap map[string]metrictypes.Type
ReducedMap map[string]bool
PromotedPathsMap map[string]bool
LogsJSONIndexes []telemetrytypes.TelemetryFieldKeySkipIndex
ColumnEvolutionMetadataMap map[string][]*telemetrytypes.EvolutionEntry
@@ -306,7 +308,7 @@ func (m *MockMetadataStore) SetAllValues(lookupKey string, values *telemetrytype
}
// FetchTemporality fetches the temporality for a metric.
func (m *MockMetadataStore) FetchTemporality(ctx context.Context, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricName string) (metrictypes.Temporality, error) {
func (m *MockMetadataStore) FetchTemporality(ctx context.Context, orgID valuer.UUID, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricName string) (metrictypes.Temporality, error) {
if temporality, exists := m.TemporalityMap[metricName]; exists {
return temporality, nil
}
@@ -314,7 +316,7 @@ func (m *MockMetadataStore) FetchTemporality(ctx context.Context, queryTimeRange
}
// FetchTemporalityMulti fetches the temporality for multiple metrics.
func (m *MockMetadataStore) FetchTemporalityMulti(ctx context.Context, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricNames ...string) (map[string]metrictypes.Temporality, error) {
func (m *MockMetadataStore) FetchTemporalityMulti(ctx context.Context, orgID valuer.UUID, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricNames ...string) (map[string]metrictypes.Temporality, error) {
result := make(map[string]metrictypes.Temporality)
for _, metricName := range metricNames {
@@ -329,9 +331,10 @@ func (m *MockMetadataStore) FetchTemporalityMulti(ctx context.Context, queryTime
}
// FetchTemporalityMulti fetches the temporality for multiple metrics.
func (m *MockMetadataStore) FetchTemporalityAndTypeMulti(ctx context.Context, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricNames ...string) (map[string]metrictypes.Temporality, map[string]metrictypes.Type, error) {
func (m *MockMetadataStore) FetchTemporalityAndTypeMulti(ctx context.Context, orgID valuer.UUID, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricNames ...string) (map[string]metrictypes.Temporality, map[string]metrictypes.Type, map[string]bool, error) {
temporalities := make(map[string]metrictypes.Temporality)
types := make(map[string]metrictypes.Type)
reduced := make(map[string]bool)
for _, metricName := range metricNames {
if temporality, exists := m.TemporalityMap[metricName]; exists {
@@ -344,9 +347,12 @@ func (m *MockMetadataStore) FetchTemporalityAndTypeMulti(ctx context.Context, qu
} else {
types[metricName] = metrictypes.UnspecifiedType
}
if m.ReducedMap[metricName] {
reduced[metricName] = true
}
}
return temporalities, types, nil
return temporalities, types, reduced, nil
}
// SetTemporality sets the temporality for a metric in the mock store.