Compare commits

..

3 Commits

Author SHA1 Message Date
nikhilmantri0902
e13cb08097 refactor(filterquery): collapse the builder into the visitor
One Visitor struct now carries sb, fmter and errors like the old
per-feature visitors did; FieldResolver stays the only new concept.
2026-09-09 20:08:33 +05:30
nikhilmantri0902
f945b5f513 refactor(dashboard): port the list filter to the shared sqlcompiler
The visitor moves to a key-policy resolver plus an error-code wrap;
emitted SQL is unchanged, pinned by the existing exact-SQL unit suite.
2026-09-09 11:33:35 +05:30
nikhilmantri0902
8b0ab0ff26 refactor(filterquery): add shared list filter SQL compiler
Extracted from the dashboards list visitor: grammar walk, operator
dispatch, value extraction, LIKE builders and the Compiled output type,
behind a per-feature FieldResolver. Scope is list pages over the
relational store; telemetry queries stay on querybuilder. Also rejects
LIKE and ILIKE patterns ending in an unescaped backslash, which never
match on sqlite and abort the query on Postgres.
2026-09-09 11:33:34 +05:30
43 changed files with 817 additions and 2846 deletions

View File

@@ -9427,8 +9427,6 @@ components:
type: string
name:
type: string
origin:
$ref: '#/components/schemas/SpantypesSpanMapperOrigin'
updatedAt:
format: date-time
type: string
@@ -9441,7 +9439,6 @@ components:
- fieldContext
- config
- enabled
- origin
type: object
SpantypesSpanMapperConfig:
properties:
@@ -9470,75 +9467,48 @@ components:
type: string
orgId:
type: string
origin:
$ref: '#/components/schemas/SpantypesSpanMapperOrigin'
updatedAt:
format: date-time
type: string
updatedBy:
type: string
version:
type: integer
required:
- id
- orgId
- name
- condition
- enabled
- origin
- version
type: object
SpantypesSpanMapperGroupCondition:
nullable: true
properties:
attributes:
items:
$ref: '#/components/schemas/SpantypesSpanMapperGroupConditionKey'
type: string
nullable: true
type: array
resource:
items:
$ref: '#/components/schemas/SpantypesSpanMapperGroupConditionKey'
type: string
nullable: true
type: array
required:
- attributes
- resource
type: object
SpantypesSpanMapperGroupConditionKey:
properties:
enabled:
type: boolean
origin:
$ref: '#/components/schemas/SpantypesSpanMapperOrigin'
value:
type: string
required:
- value
- enabled
type: object
SpantypesSpanMapperOperation:
enum:
- move
- copy
type: string
SpantypesSpanMapperOrigin:
enum:
- user
- system
type: string
SpantypesSpanMapperSource:
properties:
context:
$ref: '#/components/schemas/SpantypesFieldContext'
enabled:
type: boolean
key:
type: string
operation:
$ref: '#/components/schemas/SpantypesSpanMapperOperation'
origin:
$ref: '#/components/schemas/SpantypesSpanMapperOrigin'
priority:
type: integer
required:
@@ -9546,7 +9516,6 @@ components:
- context
- operation
- priority
- enabled
type: object
SpantypesSpanMapperTestSpan:
properties:

View File

@@ -10518,31 +10518,15 @@ export interface SpantypesGettableFlamegraphTraceDTO {
startTimestampMillis: number;
}
export enum SpantypesSpanMapperOriginDTO {
user = 'user',
system = 'system',
}
export interface SpantypesSpanMapperGroupConditionKeyDTO {
/**
* @type boolean
*/
enabled: boolean;
origin?: SpantypesSpanMapperOriginDTO;
/**
* @type string
*/
value: string;
}
export type SpantypesSpanMapperGroupConditionDTOAnyOf = {
/**
* @type array,null
*/
attributes: SpantypesSpanMapperGroupConditionKeyDTO[] | null;
attributes: string[] | null;
/**
* @type array,null
*/
resource: SpantypesSpanMapperGroupConditionKeyDTO[] | null;
resource: string[] | null;
};
/**
@@ -10578,7 +10562,6 @@ export interface SpantypesSpanMapperGroupDTO {
* @type string
*/
orgId: string;
origin: SpantypesSpanMapperOriginDTO;
/**
* @type string
* @format date-time
@@ -10588,10 +10571,6 @@ export interface SpantypesSpanMapperGroupDTO {
* @type string
*/
updatedBy?: string;
/**
* @type integer
*/
version: number;
}
export interface SpantypesGettableSpanMapperGroupsDTO {
@@ -10649,16 +10628,11 @@ export enum SpantypesSpanMapperOperationDTO {
}
export interface SpantypesSpanMapperSourceDTO {
context: SpantypesFieldContextDTO;
/**
* @type boolean
*/
enabled: boolean;
/**
* @type string
*/
key: string;
operation: SpantypesSpanMapperOperationDTO;
origin?: SpantypesSpanMapperOriginDTO;
/**
* @type integer
*/
@@ -10700,7 +10674,6 @@ export interface SpantypesSpanMapperDTO {
* @type string
*/
name: string;
origin: SpantypesSpanMapperOriginDTO;
/**
* @type string
* @format date-time

View File

@@ -333,7 +333,6 @@ describe('AttributeMappingsTab (integration)', () => {
context: FieldContext.attribute,
operation: MapperOperation.copy,
priority,
enabled: true,
})),
},
}),

View File

@@ -1,12 +1,10 @@
import { Typography } from '@signozhq/ui/typography';
import { ConditionKey } from 'container/LLMObservability/AttributeMapping/types';
import styles from './ConditionsTooltip.module.scss';
interface ConditionsTooltipProps {
attributes: ConditionKey[];
resource: ConditionKey[];
attributes: string[];
resource: string[];
}
function ConditionsTooltip({
@@ -35,8 +33,8 @@ function ConditionsTooltip({
</Typography.Text>
<div className={styles.keyList}>
{attributes.map((key) => (
<code key={`${key.origin}-${key.value}`} className={styles.key}>
{key.value}
<code key={key} className={styles.key}>
{key}
</code>
))}
</div>
@@ -49,8 +47,8 @@ function ConditionsTooltip({
</Typography.Text>
<div className={styles.keyList}>
{resource.map((key) => (
<code key={`${key.origin}-${key.value}`} className={styles.key}>
{key.value}
<code key={key} className={styles.key}>
{key}
</code>
))}
</div>

View File

@@ -3,7 +3,6 @@ import {
SpantypesSpanMapperDTO as Mapper,
SpantypesSpanMapperGroupDTO as MapperGroup,
SpantypesSpanMapperOperationDTO as MapperOperation,
SpantypesSpanMapperOriginDTO as MapperOrigin,
SpantypesSpanMapperTestSpanDTO as TestSpan,
} from 'api/generated/services/sigNoz.schemas';
@@ -22,15 +21,9 @@ export function makeGroup(overrides: Partial<MapperGroup> = {}): MapperGroup {
orgId: 'org-1',
name: 'demo',
enabled: true,
origin: MapperOrigin.user,
version: 0,
condition: {
attributes: [
{ value: 'ai.embeddings', enabled: true, origin: MapperOrigin.user },
],
resource: [
{ value: 'cloud.account.id', enabled: true, origin: MapperOrigin.user },
],
attributes: ['ai.embeddings'],
resource: ['cloud.account.id'],
},
...overrides,
};
@@ -42,7 +35,6 @@ export function makeMapper(overrides: Partial<Mapper> = {}): Mapper {
groupId: 'group-1',
name: 'gen_ai.request.model',
enabled: true,
origin: MapperOrigin.user,
fieldContext: FieldContext.attribute,
config: {
sources: [
@@ -51,16 +43,12 @@ export function makeMapper(overrides: Partial<Mapper> = {}): Mapper {
context: FieldContext.attribute,
operation: MapperOperation.copy,
priority: 2,
enabled: true,
origin: MapperOrigin.user,
},
{
key: 'llm.model',
context: FieldContext.attribute,
operation: MapperOperation.move,
priority: 1,
enabled: true,
origin: MapperOrigin.user,
},
],
},
@@ -97,12 +85,8 @@ export const mockGroups: MapperGroup[] = [
id: 'group-1',
name: 'demo',
condition: {
attributes: [
{ value: 'ai.embeddings', enabled: true, origin: MapperOrigin.user },
],
resource: [
{ value: 'cloud.account.id', enabled: true, origin: MapperOrigin.user },
],
attributes: ['ai.embeddings'],
resource: ['cloud.account.id'],
},
}),
makeGroup({

View File

@@ -1,23 +1,19 @@
import { Button } from '@signozhq/ui/button';
import { Plus, X } from '@signozhq/icons';
import {
ConditionKey,
FieldContextValue,
} from 'container/LLMObservability/AttributeMapping/types';
import { createConditionKey } from 'container/LLMObservability/AttributeMapping/utils';
import { FieldContextValue } from 'container/LLMObservability/AttributeMapping/types';
import KeySearchInput from '../../../KeySearchInput/KeySearchInput';
import styles from './ConditionKeyList.module.scss';
interface ConditionKeyListProps {
label: string;
labelHint?: string;
keys: ConditionKey[];
keys: string[];
placeholder: string;
addLabel: string;
testIdPrefix: string;
fieldContext: FieldContextValue;
onChange: (keys: ConditionKey[]) => void;
onChange: (keys: string[]) => void;
}
function ConditionKeyList({
@@ -31,11 +27,11 @@ function ConditionKeyList({
onChange,
}: ConditionKeyListProps): JSX.Element {
const updateKey = (index: number, value: string): void => {
onChange(keys.map((key, i) => (i === index ? { ...key, value } : key)));
onChange(keys.map((key, i) => (i === index ? value : key)));
};
const addKey = (): void => {
onChange([...keys, createConditionKey()]);
onChange([...keys, '']);
};
const removeKey = (index: number): void => {
@@ -57,7 +53,7 @@ function ConditionKeyList({
<KeySearchInput
className={styles.keyInput}
placeholder={placeholder}
value={key.value}
value={key}
fieldContext={fieldContext}
onChange={(next): void => updateKey(index, next)}
testId={`${testIdPrefix}-${index}`}

View File

@@ -42,9 +42,7 @@ function sourcesEqual(a: SourceConfig[], b: SourceConfig[]): boolean {
(source, index) =>
source.key === b[index].key &&
source.context === b[index].context &&
source.operation === b[index].operation &&
source.enabled === b[index].enabled &&
source.origin === b[index].origin,
source.operation === b[index].operation,
)
);
}

View File

@@ -3,7 +3,6 @@ import {
SpantypesSpanMapperDTO,
SpantypesSpanMapperGroupDTO,
SpantypesSpanMapperOperationDTO,
SpantypesSpanMapperOriginDTO,
} from 'api/generated/services/sigNoz.schemas';
export type MapperGroup = SpantypesSpanMapperGroupDTO;
@@ -12,16 +11,6 @@ export const FieldContext = SpantypesFieldContextDTO;
export type FieldContextValue = SpantypesFieldContextDTO;
export const MapperOperation = SpantypesSpanMapperOperationDTO;
export type MapperOperationValue = SpantypesSpanMapperOperationDTO;
export const MapperOrigin = SpantypesSpanMapperOriginDTO;
export type MapperOriginValue = SpantypesSpanMapperOriginDTO;
// One condition substring. Shipped (system) keys are read-only apart from
// `enabled`; user keys are fully editable.
export interface ConditionKey {
value: string;
enabled: boolean;
origin: MapperOriginValue;
}
export type MapperDraftMode = 'add' | 'edit';
@@ -29,8 +18,6 @@ export interface SourceConfig {
key: string;
context: SpantypesFieldContextDTO;
operation: SpantypesSpanMapperOperationDTO;
enabled: boolean;
origin: MapperOriginValue;
}
// Editable form state for a mapper. `sources` is ordered highest priority
@@ -46,8 +33,8 @@ export interface MapperDraft {
export interface GroupDraft {
id: string | null;
name: string;
attributes: ConditionKey[];
resource: ConditionKey[];
attributes: string[];
resource: string[];
enabled: boolean;
}
@@ -64,8 +51,8 @@ export interface DraftGroup {
localId: string;
serverId: string | null;
name: string;
attributes: ConditionKey[];
resource: ConditionKey[];
attributes: string[];
resource: string[];
enabled: boolean;
mappers: DraftMapper[];
}

View File

@@ -1,14 +1,12 @@
import {
SpantypesPostableSpanMapperDTO,
SpantypesPostableSpanMapperGroupDTO,
SpantypesSpanMapperGroupConditionKeyDTO,
SpantypesUpdatableSpanMapperDTO,
SpantypesUpdatableSpanMapperGroupDTO,
} from 'api/generated/services/sigNoz.schemas';
import { v4 as uuid } from 'uuid';
import {
ConditionKey,
DraftGroup,
DraftMapper,
FieldContext,
@@ -17,7 +15,6 @@ import {
MapperDraft,
MapperGroup,
MapperOperation,
MapperOrigin,
SourceConfig,
} from './types';
@@ -27,36 +24,20 @@ function genLocalId(prefix: 'group' | 'mapper'): string {
return `local-${prefix}-${uuid()}`;
}
export function createConditionKey(value = ''): ConditionKey {
return { value, enabled: true, origin: MapperOrigin.user };
}
// Trimmed, de-duplicated, non-empty keys preserving input order. A shipped and
// a user key may share a value, so the origin is part of the identity.
function cleanKeys(keys: ConditionKey[]): ConditionKey[] {
// Trimmed, de-duplicated, non-empty keys preserving input order.
function cleanKeys(keys: string[]): string[] {
const seen = new Set<string>();
const result: ConditionKey[] = [];
const result: string[] = [];
keys.forEach((raw) => {
const value = raw.value.trim();
const dedupeKey = `${raw.origin}:${value}`;
if (value && !seen.has(dedupeKey)) {
seen.add(dedupeKey);
result.push({ ...raw, value });
const key = raw.trim();
if (key && !seen.has(key)) {
seen.add(key);
result.push(key);
}
});
return result;
}
function fromConditionKeys(
keys: SpantypesSpanMapperGroupConditionKeyDTO[] | null | undefined,
): ConditionKey[] {
return (keys ?? []).map((key) => ({
value: key.value,
enabled: key.enabled,
origin: key.origin ?? MapperOrigin.user,
}));
}
// Source configs for a mapper, highest priority first (first match wins at
// evaluation time).
function getMapperSources(mapper: Mapper): SourceConfig[] {
@@ -67,8 +48,6 @@ function getMapperSources(mapper: Mapper): SourceConfig[] {
key: source.key,
context: source.context,
operation: source.operation,
enabled: source.enabled,
origin: source.origin ?? MapperOrigin.user,
}));
}
@@ -77,8 +56,6 @@ export function createEmptySource(): SourceConfig {
key: '',
context: FieldContext.attribute,
operation: MapperOperation.copy,
enabled: true,
origin: MapperOrigin.user,
};
}
@@ -95,7 +72,7 @@ function getCleanSources(draft: MapperDraft): SourceConfig[] {
const result: SourceConfig[] = [];
draft.sources.forEach((source) => {
const key = source.key.trim();
const dedupeKey = `${source.origin}:${source.context}:${key}`;
const dedupeKey = `${source.context}:${key}`;
if (key && !seen.has(dedupeKey)) {
seen.add(dedupeKey);
result.push({ ...source, key });
@@ -118,8 +95,6 @@ function buildSources(
context: source.context,
operation: source.operation,
priority: sources.length - index,
enabled: source.enabled,
origin: source.origin,
}));
}
@@ -148,7 +123,7 @@ export function buildUpdatableMapper(
export const EMPTY_GROUP_DRAFT: GroupDraft = {
id: null,
name: '',
attributes: [createConditionKey()],
attributes: [''],
resource: [],
enabled: true,
};
@@ -195,8 +170,8 @@ export function buildDraftGroup(
localId: group.id,
serverId: group.id,
name: group.name,
attributes: fromConditionKeys(group.condition?.attributes),
resource: fromConditionKeys(group.condition?.resource),
attributes: group.condition?.attributes ?? [],
resource: group.condition?.resource ?? [],
enabled: group.enabled,
mappers: mappers.map(buildDraftMapper),
};
@@ -207,8 +182,7 @@ export function groupDraftFromNode(group: DraftGroup): GroupDraft {
return {
id: group.localId,
name: group.name,
attributes:
group.attributes.length > 0 ? group.attributes : [createConditionKey()],
attributes: group.attributes.length > 0 ? group.attributes : [''],
resource: group.resource,
enabled: group.enabled,
};

View File

@@ -4,41 +4,17 @@ import (
"strings"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/parser/filterquery/sqlcompiler"
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
)
type Compiled struct {
SQL string
Args []any
}
func (c Compiled) IsEmpty() bool {
return c.SQL == ""
}
// Compile always returns a non-nil *Compiled. An empty query (or one that
// produces no SQL) yields a Compiled with an empty SQL — callers gate on
// SQL != "" rather than a nil check.
//
// A `key OP value` term compiles to a DSL predicate; a bare word is a
// case-insensitive substring search over the dashboard name, description, and tag
// keys/values. They compose through AND/OR/NOT, so `prod payment` matches both
// words (implicit AND) and `prod OR name = 'x'` mixes free text with a filter. A
// quoted token matches literally, e.g. `"prod payment"`.
func Compile(query string, formatter sqlstore.SQLFormatter) (*Compiled, error) {
if len(strings.TrimSpace(query)) == 0 {
return &Compiled{}, nil
}
sql, args, errs := newVisitor(formatter).compile(query)
// Compile wraps compiler errors in the dashboard list filter error code.
func Compile(query string, formatter sqlstore.SQLFormatter) (*sqlcompiler.Compiled, error) {
compiled, errs := sqlcompiler.Compile(query, formatter, dashboardFieldResolver{})
if len(errs) > 0 {
return nil, errors.NewInvalidInputf(dashboardtypes.ErrCodeDashboardListFilterInvalid,
"invalid filter query: %s", strings.Join(errs, "; "))
}
return &Compiled{
SQL: sql,
Args: args,
}, nil
return compiled, nil
}

View File

@@ -0,0 +1,125 @@
package impldashboard
import (
"strings"
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
"github.com/SigNoz/signoz/pkg/parser/filterquery/sqlcompiler"
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
qbtypesv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
sqlbuilder "github.com/huandu/go-sqlbuilder"
)
// dashboardFieldResolver maps dashboard list DSL keys; a non-reserved key is a tag key matched case-insensitively.
type dashboardFieldResolver struct{}
func (r dashboardFieldResolver) ResolveComparison(v *sqlcompiler.Visitor, rawKey string, operation qbtypesv5.FilterOperator, ctx *grammar.ComparisonContext) string {
key := strings.ToLower(rawKey)
if allowedOperations, isReserved := dashboardtypes.ReservedOps[dashboardtypes.DSLKey(key)]; isReserved {
return r.resolveReservedKey(v, ctx, operation, dashboardtypes.DSLKey(key), allowedOperations)
}
if _, allowed := dashboardtypes.TagKeyOps[operation]; !allowed {
v.AddError("operator %s is not allowed on a tag-key filter", sqlcompiler.OperationName(operation))
return ""
}
return r.tagComparison(v, ctx, operation, key)
}
func (r dashboardFieldResolver) resolveReservedKey(v *sqlcompiler.Visitor, ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, key dashboardtypes.DSLKey, allowedOperations map[qbtypesv5.FilterOperator]struct{}) string {
if _, allowed := allowedOperations[operation]; !allowed {
v.AddError("operator %s is not allowed for key %q", sqlcompiler.OperationName(operation), key)
return ""
}
switch key {
case dashboardtypes.DSLKeyName:
columnExpression := string(v.Formatter().JSONExtractString("dashboard.data", "$.spec.display.name"))
return v.StringOperation(v.SelectBuilder(), ctx, operation, columnExpression, string(key))
case dashboardtypes.DSLKeyDescription:
columnExpression := string(v.Formatter().JSONExtractString("dashboard.data", "$.spec.display.description"))
return v.StringOperation(v.SelectBuilder(), ctx, operation, columnExpression, string(key))
case dashboardtypes.DSLKeyCreatedAt:
return v.TimestampComparison(ctx, operation, "dashboard.created_at")
case dashboardtypes.DSLKeyUpdatedAt:
return v.TimestampComparison(ctx, operation, "dashboard.updated_at")
case dashboardtypes.DSLKeyCreatedBy:
return v.StringOperation(v.SelectBuilder(), ctx, operation, "dashboard.created_by", string(key))
case dashboardtypes.DSLKeyLocked:
return v.BoolComparison(ctx, operation, "dashboard.locked")
}
v.AddError("no handler for reserved key %q", key)
return ""
}
func (dashboardFieldResolver) tagComparison(v *sqlcompiler.Visitor, ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, tagKey string) string {
subqueryBuilder := sqlbuilder.NewSelectBuilder()
if operation == qbtypesv5.FilterOperatorExists || operation == qbtypesv5.FilterOperatorNotExists {
buildSubqueryForTagKey(subqueryBuilder, tagKey)
} else {
// Value predicates take the positive operator; negation toggles the EXISTS wrapper.
positiveOperation := operation
if operation.IsNegativeOperator() {
positiveOperation = operation.Inverse()
}
valuePredicate := v.StringOperation(subqueryBuilder, ctx, positiveOperation, "t.value", tagKey)
if valuePredicate == "" {
return ""
}
buildSubqueryForTagKeyAndValue(subqueryBuilder, tagKey, valuePredicate)
}
if operation.IsNegativeOperator() {
return v.SelectBuilder().NotExists(subqueryBuilder)
}
return v.SelectBuilder().Exists(subqueryBuilder)
}
func buildSubqueryForTagKey(subqueryBuilder *sqlbuilder.SelectBuilder, tagKey string) *sqlbuilder.SelectBuilder {
const dashboardTagKind = `"dashboard"`
return subqueryBuilder.
Select("1").
From("tag_relation tr").
Join("tag t", "t.id = tr.tag_id").
Where(
subqueryBuilder.Equal("tr.kind", dashboardTagKind),
"tr.resource_id = dashboard.id",
"LOWER(t.key) = LOWER("+subqueryBuilder.Var(tagKey)+")",
)
}
func buildSubqueryForTagKeyAndValue(subqueryBuilder *sqlbuilder.SelectBuilder, tagKey, valuePredicate string) *sqlbuilder.SelectBuilder {
return buildSubqueryForTagKey(subqueryBuilder, tagKey).Where(valuePredicate)
}
// FreeText searches name, description and tag keys/values.
func (dashboardFieldResolver) FreeText(v *sqlcompiler.Visitor, value string) string {
nameColumn := string(v.Formatter().JSONExtractString("dashboard.data", "$.spec.display.name"))
descriptionColumn := string(v.Formatter().JSONExtractString("dashboard.data", "$.spec.display.description"))
namePredicate := v.FreeTextContains(v.SelectBuilder(), nameColumn, value)
descriptionPredicate := v.FreeTextContains(v.SelectBuilder(), descriptionColumn, value)
subqueryBuilder := sqlbuilder.NewSelectBuilder()
keyPredicate := v.FreeTextContains(subqueryBuilder, "t.key", value)
valuePredicate := v.FreeTextContains(subqueryBuilder, "t.value", value)
buildSubqueryForFreeTextTag(subqueryBuilder, keyPredicate, valuePredicate)
tagPredicate := v.SelectBuilder().Exists(subqueryBuilder)
return v.SelectBuilder().Or(namePredicate, descriptionPredicate, tagPredicate)
}
func buildSubqueryForFreeTextTag(subqueryBuilder *sqlbuilder.SelectBuilder, keyPredicate, valuePredicate string) *sqlbuilder.SelectBuilder {
const dashboardTagKind = `"dashboard"`
return subqueryBuilder.
Select("1").
From("tag_relation tr").
Join("tag t", "t.id = tr.tag_id").
Where(
subqueryBuilder.Equal("tr.kind", dashboardTagKind),
"tr.resource_id = dashboard.id",
subqueryBuilder.Or(keyPredicate, valuePredicate),
)
}

View File

@@ -559,6 +559,11 @@ func TestCompile_Rejections(t *testing.T) {
dslQueryToCompile: `created_at >= 'not-a-date'`,
expectedErrShouldContain: "RFC3339",
},
{
subtestName: "rejects LIKE pattern ending in an unescaped backslash",
dslQueryToCompile: `name LIKE 'prod\\'`,
expectedErrShouldContain: "must not end with an unescaped backslash",
},
{
subtestName: "rejects REGEXP — not yet supported",
dslQueryToCompile: `name REGEXP '.*'`,
@@ -573,7 +578,7 @@ func TestCompile_Rejections(t *testing.T) {
}
// Every key in dashboardtypes.ReservedOps must have a matching case in
// visitComparisonForReservedKeys; a key that's reserved but unhandled falls
// resolveReservedKey; a key that's reserved but unhandled falls
// through to the "no handler for reserved key" error. Equal is accepted by all
// reserved keys, so `key = 'x'` always reaches the dispatch switch — a missing
// handler surfaces as that error regardless of whether the value type-checks.
@@ -583,7 +588,7 @@ func TestCompileReservedKeysAllHandled(t *testing.T) {
_, err := Compile(string(key)+` = 'x'`, formatter(t))
if err != nil {
assert.NotContains(t, err.Error(), "no handler for reserved key",
"reserved key %q has no handler in visitComparisonForReservedKeys", key)
"reserved key %q has no handler in resolveReservedKey", key)
}
})
}

View File

@@ -1,631 +0,0 @@
package impldashboard
import (
"fmt"
"strings"
"time"
"github.com/SigNoz/signoz/pkg/parser/filterquery"
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
qbtypesv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/antlr4-go/antlr/v4"
sqlbuilder "github.com/huandu/go-sqlbuilder"
)
// bunPlaceholderFlavor is any flavor that renders `?` placeholders, which bun
// re-binds to the actual backend (e.g. `$1` for Postgres) at query time.
const bunPlaceholderFlavor = sqlbuilder.SQLite
type visitor struct {
grammar.BaseFilterQueryVisitor
selectBuilder *sqlbuilder.SelectBuilder
formatter sqlstore.SQLFormatter
errors []string
}
func newVisitor(formatter sqlstore.SQLFormatter) *visitor {
return &visitor{
selectBuilder: sqlbuilder.NewSelectBuilder(),
formatter: formatter,
}
}
// compile builds `?`-placeholder WHERE SQL + args for bun. Each term is either a
// `key OP value` comparison or a bare token that becomes a free-text search; the
// two compose through the boolean grammar (AND/OR/NOT). Malformed input is
// returned as errors.
func (v *visitor) compile(query string) (string, []any, []string) {
tree, _, collector := filterquery.Parse(query)
if len(collector.Errors) > 0 {
return "", nil, collector.Errors
}
condition, _ := v.visit(tree).(string)
if len(v.errors) > 0 {
return "", nil, v.errors
}
if condition == "" {
return "", nil, nil
}
sql, arguments := v.selectBuilder.Args.CompileWithFlavor(condition, bunPlaceholderFlavor)
return sql, arguments, nil
}
func (v *visitor) visit(tree antlr.ParseTree) any {
if tree == nil {
return nil
}
return tree.Accept(v)
}
// ════════════════════════════════════════════════════════════════════════
// methods from grammar.BaseFilterQueryVisitor that are overridden
// ════════════════════════════════════════════════════════════════════════
func (v *visitor) VisitQuery(ctx *grammar.QueryContext) any {
return v.visit(ctx.Expression())
}
func (v *visitor) VisitExpression(ctx *grammar.ExpressionContext) any {
return v.visit(ctx.OrExpression())
}
func (v *visitor) VisitOrExpression(ctx *grammar.OrExpressionContext) any {
parts := ctx.AllAndExpression()
conditions := make([]string, 0, len(parts))
for _, part := range parts {
if condition, ok := v.visit(part).(string); ok && condition != "" {
conditions = append(conditions, condition)
}
}
switch len(conditions) {
case 0:
return ""
case 1:
return conditions[0]
default:
return v.selectBuilder.Or(conditions...)
}
}
func (v *visitor) VisitAndExpression(ctx *grammar.AndExpressionContext) any {
parts := ctx.AllUnaryExpression()
conditions := make([]string, 0, len(parts))
for _, part := range parts {
if condition, ok := v.visit(part).(string); ok && condition != "" {
conditions = append(conditions, condition)
}
}
switch len(conditions) {
case 0:
return ""
case 1:
return conditions[0]
default:
return v.selectBuilder.And(conditions...)
}
}
func (v *visitor) VisitUnaryExpression(ctx *grammar.UnaryExpressionContext) any {
condition, _ := v.visit(ctx.Primary()).(string)
if condition == "" {
return ""
}
if ctx.NOT() != nil {
return fmt.Sprintf("NOT (%s)", condition)
}
return condition
}
func (v *visitor) VisitPrimary(ctx *grammar.PrimaryContext) any {
if ctx.OrExpression() != nil {
return v.visit(ctx.OrExpression())
}
if ctx.Comparison() != nil {
return v.visit(ctx.Comparison())
}
// A lone key/value/full-text token is a free-text term, composed with any
// comparisons through the boolean grammar. A quoted token matches its contents
// literally — the escape hatch for a phrase or a term that looks like DSL.
return v.buildFreeTextTerm(trimQuotes(ctx.GetText()))
}
// VisitComparison dispatches a single `key OP value` term. A key that matches
// a reserved DSL key (name, description, etc.) becomes a column-level
// predicate; any other identifier is treated as a tag key — the operator
// applies to the tag's value, with a case-insensitive match on the tag's key.
func (v *visitor) VisitComparison(ctx *grammar.ComparisonContext) any {
key := strings.ToLower(strings.TrimSpace(ctx.Key().GetText()))
operation, ok := v.extractOperation(ctx)
if !ok {
return ""
}
if allowedOperations, isReserved := dashboardtypes.ReservedOps[dashboardtypes.DSLKey(key)]; isReserved {
return v.visitComparisonForReservedKeys(ctx, operation, dashboardtypes.DSLKey(key), allowedOperations)
}
return v.visitComparisonForTags(ctx, operation, key)
}
func (v *visitor) visitComparisonForReservedKeys(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, key dashboardtypes.DSLKey, allowedOperations map[qbtypesv5.FilterOperator]struct{}) string {
if _, allowed := allowedOperations[operation]; !allowed {
v.addError("operator %s is not allowed for key %q", operationName(operation), key)
return ""
}
switch key {
case dashboardtypes.DSLKeyName:
return v.buildJSONStringComparison(ctx, operation, dashboardtypes.DSLKeyName, "$.spec.display.name")
case dashboardtypes.DSLKeyDescription:
return v.buildJSONStringComparison(ctx, operation, dashboardtypes.DSLKeyDescription, "$.spec.display.description")
case dashboardtypes.DSLKeyCreatedAt:
return v.buildTimestampComparison(ctx, operation, "dashboard.created_at")
case dashboardtypes.DSLKeyUpdatedAt:
return v.buildTimestampComparison(ctx, operation, "dashboard.updated_at")
case dashboardtypes.DSLKeyCreatedBy:
return v.buildStringComparison(ctx, operation, dashboardtypes.DSLKeyCreatedBy, "dashboard.created_by")
case dashboardtypes.DSLKeyLocked:
return v.buildBoolComparison(ctx, operation, "dashboard.locked")
}
// Unreachable for real input: every dashboardtypes.ReservedOps key has a case above, and
// TestCompileReservedKeysAllHandled guards that the two stay in sync.
v.addError("no handler for reserved key %q", key)
return ""
}
func (v *visitor) visitComparisonForTags(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, tagKey string) string {
if _, allowed := dashboardtypes.TagKeyOps[operation]; !allowed {
v.addError("operator %s is not allowed on a tag-key filter", operationName(operation))
return ""
}
return v.buildTagComparison(ctx, operation, tagKey)
}
func (v *visitor) extractOperation(ctx *grammar.ComparisonContext) (qbtypesv5.FilterOperator, bool) {
// For operators that take an optional leading NOT, Inverse() maps each to
// its Not<X> counterpart.
maybeNot := func(operation qbtypesv5.FilterOperator) qbtypesv5.FilterOperator {
if ctx.NOT() != nil {
return operation.Inverse()
}
return operation
}
switch {
case ctx.EQUALS() != nil:
return qbtypesv5.FilterOperatorEqual, true
case ctx.NOT_EQUALS() != nil, ctx.NEQ() != nil:
return qbtypesv5.FilterOperatorNotEqual, true
case ctx.LT() != nil:
return qbtypesv5.FilterOperatorLessThan, true
case ctx.LE() != nil:
return qbtypesv5.FilterOperatorLessThanOrEq, true
case ctx.GT() != nil:
return qbtypesv5.FilterOperatorGreaterThan, true
case ctx.GE() != nil:
return qbtypesv5.FilterOperatorGreaterThanOrEq, true
case ctx.BETWEEN() != nil:
return maybeNot(qbtypesv5.FilterOperatorBetween), true
case ctx.LIKE() != nil:
return maybeNot(qbtypesv5.FilterOperatorLike), true
case ctx.ILIKE() != nil:
return maybeNot(qbtypesv5.FilterOperatorILike), true
case ctx.CONTAINS() != nil:
return maybeNot(qbtypesv5.FilterOperatorContains), true
case ctx.REGEXP() != nil:
return maybeNot(qbtypesv5.FilterOperatorRegexp), true
case ctx.InClause() != nil:
return qbtypesv5.FilterOperatorIn, true
case ctx.NotInClause() != nil:
return qbtypesv5.FilterOperatorNotIn, true
case ctx.EXISTS() != nil:
return maybeNot(qbtypesv5.FilterOperatorExists), true
}
v.addError("could not determine operator in expression %q", ctx.GetText())
return qbtypesv5.FilterOperatorUnknown, false
}
// ─── per-key emitters ────────────────────────────────────────────────────────
func (v *visitor) buildJSONStringComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, key dashboardtypes.DSLKey, jsonPath string) string {
columnExpression := string(v.formatter.JSONExtractString("dashboard.data", jsonPath))
return v.buildStringOperation(v.selectBuilder, ctx, operation, columnExpression, string(key))
}
func (v *visitor) buildStringComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, key dashboardtypes.DSLKey, columnExpression string) string {
return v.buildStringOperation(v.selectBuilder, ctx, operation, columnExpression, string(key))
}
// buildStringOperation covers all the operators the spec allows on text-shaped keys
// (name, description, created_by, and a tag's value). Placeholders are interned
// into builder — the outer builder for column predicates, the subquery builder for
// tag-value predicates — so nested EXISTS arguments thread correctly.
func (v *visitor) buildStringOperation(builder *sqlbuilder.SelectBuilder, ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression, keyForError string) string {
switch operation {
case qbtypesv5.FilterOperatorEqual:
val, ok := v.extractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
return builder.Equal(columnExpression, val)
case qbtypesv5.FilterOperatorNotEqual:
val, ok := v.extractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
return builder.NotEqual(columnExpression, val)
case qbtypesv5.FilterOperatorLike, qbtypesv5.FilterOperatorNotLike:
val, ok := v.extractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
like := "LIKE"
if operation == qbtypesv5.FilterOperatorNotLike {
like = "NOT LIKE"
}
// The user's % and _ stay as wildcards; ESCAPE pins backslash as the escape
// char so a literal `\` in the pattern is read the same on both dialects —
// Postgres defaults to `\`, SQLite has no default escape.
return fmt.Sprintf("%s %s %s ESCAPE '\\'", columnExpression, like, builder.Var(val))
case qbtypesv5.FilterOperatorILike, qbtypesv5.FilterOperatorNotILike:
val, ok := v.extractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
// SQLite has no ILIKE keyword and Postgres LIKE is case-sensitive — emit
// LOWER(col) LIKE LOWER(?) so behavior is identical on both dialects. ESCAPE
// pins backslash as the escape char (Postgres default; SQLite has none).
lowerColumn := string(v.formatter.LowerExpression(columnExpression))
like := "LIKE"
if operation == qbtypesv5.FilterOperatorNotILike {
like = "NOT LIKE"
}
return fmt.Sprintf("%s %s LOWER(%s) ESCAPE '\\'", lowerColumn, like, builder.Var(val))
case qbtypesv5.FilterOperatorContains, qbtypesv5.FilterOperatorNotContains:
val, ok := v.extractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
like := "LIKE"
if operation == qbtypesv5.FilterOperatorNotContains {
like = "NOT LIKE"
}
// Escape the user's % and _ so they match literally, then wrap in wildcards.
// ESCAPE declares the backslash the escaper injected as the escape char —
// needed on SQLite (no default) and a harmless restatement of the Postgres default.
escaped := v.formatter.EscapeLikePattern(val)
return fmt.Sprintf("%s %s %s ESCAPE '\\'", columnExpression, like, builder.Var("%"+escaped+"%"))
case qbtypesv5.FilterOperatorRegexp, qbtypesv5.FilterOperatorNotRegexp:
v.addError("REGEXP filtering on %q is not yet supported", keyForError)
return ""
case qbtypesv5.FilterOperatorIn, qbtypesv5.FilterOperatorNotIn:
values, ok := v.extractStringValueList(ctx, keyForError)
if !ok {
return ""
}
arguments := make([]any, len(values))
for i, s := range values {
arguments[i] = s
}
if operation == qbtypesv5.FilterOperatorNotIn {
return builder.NotIn(columnExpression, arguments...)
}
return builder.In(columnExpression, arguments...)
}
v.addError("operator %s on %q is not implemented", operationName(operation), keyForError)
return ""
}
func (v *visitor) buildTimestampComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression string) string {
switch operation {
case qbtypesv5.FilterOperatorEqual, qbtypesv5.FilterOperatorNotEqual,
qbtypesv5.FilterOperatorLessThan, qbtypesv5.FilterOperatorLessThanOrEq,
qbtypesv5.FilterOperatorGreaterThan, qbtypesv5.FilterOperatorGreaterThanOrEq:
t, ok := v.extractSingleTimestampValue(ctx)
if !ok {
return ""
}
switch operation {
case qbtypesv5.FilterOperatorEqual:
return v.selectBuilder.Equal(columnExpression, t)
case qbtypesv5.FilterOperatorNotEqual:
return v.selectBuilder.NotEqual(columnExpression, t)
case qbtypesv5.FilterOperatorLessThan:
return v.selectBuilder.LessThan(columnExpression, t)
case qbtypesv5.FilterOperatorLessThanOrEq:
return v.selectBuilder.LessEqualThan(columnExpression, t)
case qbtypesv5.FilterOperatorGreaterThan:
return v.selectBuilder.GreaterThan(columnExpression, t)
case qbtypesv5.FilterOperatorGreaterThanOrEq:
return v.selectBuilder.GreaterEqualThan(columnExpression, t)
}
case qbtypesv5.FilterOperatorBetween, qbtypesv5.FilterOperatorNotBetween:
timestamps, ok := v.extractTwoTimestampValues(ctx)
if !ok {
return ""
}
if operation == qbtypesv5.FilterOperatorNotBetween {
return v.selectBuilder.NotBetween(columnExpression, timestamps[0], timestamps[1])
}
return v.selectBuilder.Between(columnExpression, timestamps[0], timestamps[1])
}
v.addError("operator %s on timestamp is not implemented", operationName(operation))
return ""
}
func (v *visitor) buildBoolComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression string) string {
b, ok := v.extractSingleBoolValue(ctx)
if !ok {
return ""
}
if operation == qbtypesv5.FilterOperatorNotEqual {
return v.selectBuilder.NotEqual(columnExpression, b)
}
return v.selectBuilder.Equal(columnExpression, b)
}
func (v *visitor) buildTagComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, tagKey string) string {
subqueryBuilder := sqlbuilder.NewSelectBuilder()
if operation == qbtypesv5.FilterOperatorExists || operation == qbtypesv5.FilterOperatorNotExists {
buildSubqueryForTagKey(subqueryBuilder, tagKey)
} else {
// All other tag operators take the positive form of the value predicate
// and toggle the EXISTS wrapper for negation. Inverse() flips Not<X> → <X>.
positiveOperation := operation
if operation.IsNegativeOperator() {
positiveOperation = operation.Inverse()
}
valuePredicate := v.buildStringOperation(subqueryBuilder, ctx, positiveOperation, "t.value", tagKey)
if valuePredicate == "" {
return ""
}
buildSubqueryForTagKeyAndValue(subqueryBuilder, tagKey, valuePredicate)
}
if operation.IsNegativeOperator() {
return v.selectBuilder.NotExists(subqueryBuilder)
}
return v.selectBuilder.Exists(subqueryBuilder)
}
func buildSubqueryForTagKey(subqueryBuilder *sqlbuilder.SelectBuilder, tagKey string) *sqlbuilder.SelectBuilder {
const dashboardTagKind = `"dashboard"`
return subqueryBuilder.
Select("1").
From("tag_relation tr").
Join("tag t", "t.id = tr.tag_id").
Where(
subqueryBuilder.Equal("tr.kind", dashboardTagKind),
"tr.resource_id = dashboard.id",
"LOWER(t.key) = LOWER("+subqueryBuilder.Var(tagKey)+")",
)
}
func buildSubqueryForTagKeyAndValue(subqueryBuilder *sqlbuilder.SelectBuilder, tagKey, valuePredicate string) *sqlbuilder.SelectBuilder {
return buildSubqueryForTagKey(subqueryBuilder, tagKey).Where(valuePredicate)
}
// ─── free-text search ────────────────────────────────────────────────────────
// buildFreeTextTerm matches value as a case-insensitive substring of the
// dashboard name, description, or any tag key/value.
func (v *visitor) buildFreeTextTerm(value string) string {
nameColumn := string(v.formatter.JSONExtractString("dashboard.data", "$.spec.display.name"))
descriptionColumn := string(v.formatter.JSONExtractString("dashboard.data", "$.spec.display.description"))
namePredicate := v.buildFreeTextContains(v.selectBuilder, nameColumn, value)
descriptionPredicate := v.buildFreeTextContains(v.selectBuilder, descriptionColumn, value)
subqueryBuilder := sqlbuilder.NewSelectBuilder()
keyPredicate := v.buildFreeTextContains(subqueryBuilder, "t.key", value)
valuePredicate := v.buildFreeTextContains(subqueryBuilder, "t.value", value)
buildSubqueryForFreeTextTag(subqueryBuilder, keyPredicate, valuePredicate)
tagPredicate := v.selectBuilder.Exists(subqueryBuilder)
return v.selectBuilder.Or(namePredicate, descriptionPredicate, tagPredicate)
}
// buildFreeTextContains emits a case-insensitive contains as
// LOWER(COALESCE(col, '')) LIKE LOWER(?), identical on SQLite and Postgres.
// COALESCE keeps a NULL column (an absent description) false rather than NULL —
// otherwise `NOT (…)` goes NULL and drops every description-less dashboard. The
// value's % and _ are escaped, and ESCAPE pins backslash as the escape char.
func (v *visitor) buildFreeTextContains(builder *sqlbuilder.SelectBuilder, columnExpression, value string) string {
lowerColumn := string(v.formatter.LowerExpression("COALESCE(" + columnExpression + ", '')"))
pattern := "%" + v.formatter.EscapeLikePattern(value) + "%"
return fmt.Sprintf("%s LIKE LOWER(%s) ESCAPE '\\'", lowerColumn, builder.Var(pattern))
}
func buildSubqueryForFreeTextTag(subqueryBuilder *sqlbuilder.SelectBuilder, keyPredicate, valuePredicate string) *sqlbuilder.SelectBuilder {
const dashboardTagKind = `"dashboard"`
return subqueryBuilder.
Select("1").
From("tag_relation tr").
Join("tag t", "t.id = tr.tag_id").
Where(
subqueryBuilder.Equal("tr.kind", dashboardTagKind),
"tr.resource_id = dashboard.id",
subqueryBuilder.Or(keyPredicate, valuePredicate),
)
}
// ─── value extraction helpers ───────────────────────────────────────────────
func (v *visitor) addError(format string, arguments ...any) {
v.errors = append(v.errors, fmt.Sprintf(format, arguments...))
}
func (v *visitor) extractSingleStringValue(ctx *grammar.ComparisonContext, keyForError string) (string, bool) {
values := ctx.AllValue()
if len(values) != 1 {
v.addError("expected exactly one value for %q", keyForError)
return "", false
}
return v.extractStringValue(values[0], keyForError)
}
func (v *visitor) extractSingleBoolValue(ctx *grammar.ComparisonContext) (bool, bool) {
values := ctx.AllValue()
if len(values) != 1 {
v.addError("expected a single boolean (true/false)")
return false, false
}
return v.extractBoolValue(values[0])
}
func (v *visitor) extractSingleTimestampValue(ctx *grammar.ComparisonContext) (time.Time, bool) {
values := ctx.AllValue()
if len(values) != 1 {
v.addError("expected a single RFC3339 timestamp")
return time.Time{}, false
}
return v.extractTimestampValue(values[0])
}
func (v *visitor) extractTwoTimestampValues(ctx *grammar.ComparisonContext) ([2]time.Time, bool) {
values := ctx.AllValue()
if len(values) != 2 {
v.addError("BETWEEN expects two RFC3339 timestamps")
return [2]time.Time{}, false
}
a, ok1 := v.extractTimestampValue(values[0])
b, ok2 := v.extractTimestampValue(values[1])
if !ok1 || !ok2 {
return [2]time.Time{}, false
}
return [2]time.Time{a, b}, true
}
func (v *visitor) extractStringValueList(ctx *grammar.ComparisonContext, keyForError string) ([]string, bool) {
var valuesCtx []grammar.IValueContext
switch {
case ctx.InClause() != nil:
inClause := ctx.InClause()
if inClause.ValueList() != nil {
valuesCtx = inClause.ValueList().AllValue()
} else {
valuesCtx = []grammar.IValueContext{inClause.Value()}
}
case ctx.NotInClause() != nil:
notInClause := ctx.NotInClause()
if notInClause.ValueList() != nil {
valuesCtx = notInClause.ValueList().AllValue()
} else {
valuesCtx = []grammar.IValueContext{notInClause.Value()}
}
default:
v.addError("IN clause is missing for %q", keyForError)
return nil, false
}
if len(valuesCtx) == 0 {
v.addError("IN list for %q is empty", keyForError)
return nil, false
}
out := make([]string, 0, len(valuesCtx))
for _, valueContext := range valuesCtx {
s, ok := v.extractStringValue(valueContext, keyForError)
if !ok {
return nil, false
}
out = append(out, s)
}
return out, true
}
func (v *visitor) extractStringValue(ctx grammar.IValueContext, keyForError string) (string, bool) {
if ctx.QUOTED_TEXT() != nil {
return trimQuotes(ctx.QUOTED_TEXT().GetText()), true
}
if ctx.KEY() != nil {
// Bare tokens are accepted as strings, mirroring the FilterQuery lexer's
// treatment of unquoted identifiers on the value side.
return ctx.KEY().GetText(), true
}
v.addError("expected a string value for %q, got %q", keyForError, ctx.GetText())
return "", false
}
func (v *visitor) extractBoolValue(ctx grammar.IValueContext) (bool, bool) {
if ctx.BOOL() == nil {
v.addError("expected a boolean (true/false), got %q", ctx.GetText())
return false, false
}
return strings.EqualFold(ctx.BOOL().GetText(), "true"), true
}
func (v *visitor) extractTimestampValue(ctx grammar.IValueContext) (time.Time, bool) {
if ctx.QUOTED_TEXT() == nil {
v.addError("expected an RFC3339 timestamp string, got %q", ctx.GetText())
return time.Time{}, false
}
raw := trimQuotes(ctx.QUOTED_TEXT().GetText())
t, err := time.Parse(time.RFC3339, raw)
if err != nil {
v.addError("invalid RFC3339 timestamp %q: %s", raw, err.Error())
return time.Time{}, false
}
return t, true
}
// ─── operator spelling ───────────────────────────────────────────────────────
// operationName returns the user-facing spelling of a FilterOperator, used only in
// error messages — go-sqlbuilder's Cond helpers emit the SQL keywords.
func operationName(operation qbtypesv5.FilterOperator) string {
switch operation {
case qbtypesv5.FilterOperatorEqual:
return "="
case qbtypesv5.FilterOperatorNotEqual:
return "!="
case qbtypesv5.FilterOperatorLessThan:
return "<"
case qbtypesv5.FilterOperatorLessThanOrEq:
return "<="
case qbtypesv5.FilterOperatorGreaterThan:
return ">"
case qbtypesv5.FilterOperatorGreaterThanOrEq:
return ">="
case qbtypesv5.FilterOperatorBetween:
return "BETWEEN"
case qbtypesv5.FilterOperatorNotBetween:
return "NOT BETWEEN"
case qbtypesv5.FilterOperatorLike:
return "LIKE"
case qbtypesv5.FilterOperatorNotLike:
return "NOT LIKE"
case qbtypesv5.FilterOperatorILike:
return "ILIKE"
case qbtypesv5.FilterOperatorNotILike:
return "NOT ILIKE"
case qbtypesv5.FilterOperatorContains:
return "CONTAINS"
case qbtypesv5.FilterOperatorNotContains:
return "NOT CONTAINS"
case qbtypesv5.FilterOperatorRegexp:
return "REGEXP"
case qbtypesv5.FilterOperatorNotRegexp:
return "NOT REGEXP"
case qbtypesv5.FilterOperatorIn:
return "IN"
case qbtypesv5.FilterOperatorNotIn:
return "NOT IN"
case qbtypesv5.FilterOperatorExists:
return "EXISTS"
case qbtypesv5.FilterOperatorNotExists:
return "NOT EXISTS"
}
return "?"
}
func trimQuotes(s string) string {
if len(s) >= 2 {
if (s[0] == '"' && s[len(s)-1] == '"') || (s[0] == '\'' && s[len(s)-1] == '\'') {
s = s[1 : len(s)-1]
}
}
s = strings.ReplaceAll(s, `\\`, `\`)
s = strings.ReplaceAll(s, `\'`, `'`)
return s
}

View File

@@ -7,7 +7,6 @@ import (
"github.com/SigNoz/signoz/pkg/modules/dashboard"
"github.com/SigNoz/signoz/pkg/modules/organization"
"github.com/SigNoz/signoz/pkg/modules/quickfilter"
"github.com/SigNoz/signoz/pkg/modules/spanmapper"
"github.com/SigNoz/signoz/pkg/types"
"github.com/SigNoz/signoz/pkg/valuer"
)
@@ -17,11 +16,10 @@ type setter struct {
alertmanager alertmanager.Alertmanager
quickfilter quickfilter.Module
dashboard dashboard.Module
spanMapper spanmapper.Module
}
func NewSetter(store types.OrganizationStore, alertmanager alertmanager.Alertmanager, quickfilter quickfilter.Module, dashboard dashboard.Module, spanMapper spanmapper.Module) organization.Setter {
return &setter{store: store, alertmanager: alertmanager, quickfilter: quickfilter, dashboard: dashboard, spanMapper: spanMapper}
func NewSetter(store types.OrganizationStore, alertmanager alertmanager.Alertmanager, quickfilter quickfilter.Module, dashboard dashboard.Module) organization.Setter {
return &setter{store: store, alertmanager: alertmanager, quickfilter: quickfilter, dashboard: dashboard}
}
func (module *setter) Create(ctx context.Context, organization *types.Organization, createManagedRoles func(context.Context, valuer.UUID) error) error {
@@ -45,10 +43,6 @@ func (module *setter) Create(ctx context.Context, organization *types.Organizati
return err
}
if err := module.spanMapper.ReconcileSystemGroups(ctx, organization.ID); err != nil {
return err
}
return nil
}

View File

@@ -1,46 +0,0 @@
package implspanmapper
import (
"embed"
"io/fs"
"path"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types/spantypes"
)
const definitionsRoot = "fs/definitions"
//go:embed fs/definitions/*.json
var definitionFiles embed.FS
// NewSystemGroupRegistry parses every embedded definition. Definitions are
// build-time assets validated by a test, so a failure here means the binary
// shipped broken JSON.
func NewSystemGroupRegistry() (spantypes.SpanMapperGroupRegistry, error) {
entries, err := fs.ReadDir(definitionFiles, definitionsRoot)
if err != nil {
return spantypes.SpanMapperGroupRegistry{}, errors.WrapInternalf(err, errors.CodeInternal, "couldn't read span mapper group definitions")
}
definitions := make([]spantypes.SpanMapperGroupDefinition, 0, len(entries))
for _, entry := range entries {
if entry.IsDir() {
continue
}
file := path.Join(definitionsRoot, entry.Name())
raw, err := definitionFiles.ReadFile(file)
if err != nil {
return spantypes.SpanMapperGroupRegistry{}, errors.WrapInternalf(err, errors.CodeInternal, "couldn't read %s", file)
}
definition, err := spantypes.NewSpanMapperGroupDefinition(raw)
if err != nil {
return spantypes.SpanMapperGroupRegistry{}, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "couldn't parse %s", file)
}
definitions = append(definitions, definition)
}
return spantypes.NewSpanMapperGroupRegistry(definitions)
}

View File

@@ -1,79 +0,0 @@
{
"version": 1,
"definition": {
"name": "agent",
"condition": {
"attributes": [
{
"value": "agent"
}
],
"resource": []
},
"enabled": true,
"mappers": [
{
"name": "gen_ai.agent.name",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "agent.name",
"context": "attribute",
"operation": "copy",
"priority": 20
},
{
"key": "agent_name",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.agent.id",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "agent.id",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.agent.description",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "agent.description",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.output.messages",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "final_result",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
}
]
}
}

View File

@@ -1,347 +0,0 @@
{
"version": 1,
"definition": {
"name": "llm",
"condition": {
"attributes": [
{
"value": "model"
}
],
"resource": []
},
"enabled": true,
"mappers": [
{
"name": "gen_ai.request.model",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "llm.model_name",
"context": "attribute",
"operation": "copy",
"priority": 60
},
{
"key": "llm.request.model",
"context": "attribute",
"operation": "copy",
"priority": 50
},
{
"key": "ai.model.id",
"context": "attribute",
"operation": "copy",
"priority": 40
},
{
"key": "langfuse.observation.model.name",
"context": "attribute",
"operation": "copy",
"priority": 30
},
{
"key": "embedding.model_name",
"context": "attribute",
"operation": "copy",
"priority": 20
},
{
"key": "model",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.response.model",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "llm.response.model",
"context": "attribute",
"operation": "copy",
"priority": 20
},
{
"key": "ai.response.model",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.provider.name",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "gen_ai.system",
"context": "attribute",
"operation": "copy",
"priority": 50
},
{
"key": "llm.vendor",
"context": "attribute",
"operation": "copy",
"priority": 40
},
{
"key": "llm.provider",
"context": "attribute",
"operation": "copy",
"priority": 30
},
{
"key": "llm.system",
"context": "attribute",
"operation": "copy",
"priority": 20
},
{
"key": "ai.model.provider",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.operation.name",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "llm.request.type",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.usage.input_tokens",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "gen_ai.usage.prompt_tokens",
"context": "attribute",
"operation": "copy",
"priority": 50
},
{
"key": "llm.usage.prompt_tokens",
"context": "attribute",
"operation": "copy",
"priority": 40
},
{
"key": "llm.token_count.prompt",
"context": "attribute",
"operation": "copy",
"priority": 30
},
{
"key": "ai.usage.inputTokens",
"context": "attribute",
"operation": "copy",
"priority": 20
},
{
"key": "ai.usage.promptTokens",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.usage.output_tokens",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "gen_ai.usage.completion_tokens",
"context": "attribute",
"operation": "copy",
"priority": 50
},
{
"key": "llm.usage.completion_tokens",
"context": "attribute",
"operation": "copy",
"priority": 40
},
{
"key": "llm.token_count.completion",
"context": "attribute",
"operation": "copy",
"priority": 30
},
{
"key": "ai.usage.outputTokens",
"context": "attribute",
"operation": "copy",
"priority": 20
},
{
"key": "ai.usage.completionTokens",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.usage.cache_read.input_tokens",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "gen_ai.usage.cache_read_input_tokens",
"context": "attribute",
"operation": "copy",
"priority": 30
},
{
"key": "llm.token_count.prompt_details.cache_read",
"context": "attribute",
"operation": "copy",
"priority": 20
},
{
"key": "ai.usage.cachedInputTokens",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.usage.cache_creation.input_tokens",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "gen_ai.usage.cache_write.input_tokens",
"context": "attribute",
"operation": "copy",
"priority": 30
},
{
"key": "gen_ai.usage.cache_creation_input_tokens",
"context": "attribute",
"operation": "copy",
"priority": 20
},
{
"key": "llm.token_count.prompt_details.cache_write",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.input.messages",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "gen_ai.prompt",
"context": "attribute",
"operation": "copy",
"priority": 30
},
{
"key": "ai.prompt.messages",
"context": "attribute",
"operation": "copy",
"priority": 20
},
{
"key": "input.value",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.output.messages",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "gen_ai.completion",
"context": "attribute",
"operation": "copy",
"priority": 30
},
{
"key": "ai.response.text",
"context": "attribute",
"operation": "copy",
"priority": 20
},
{
"key": "output.value",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.conversation.id",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "session.id",
"context": "attribute",
"operation": "copy",
"priority": 20
},
{
"key": "langfuse.session.id",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.response.finish_reason",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "ai.response.finishReason",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
}
]
}
}

View File

@@ -1,135 +0,0 @@
{
"version": 1,
"definition": {
"name": "tool",
"condition": {
"attributes": [
{
"value": "tool"
}
],
"resource": []
},
"enabled": true,
"mappers": [
{
"name": "gen_ai.tool.name",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "tool.name",
"context": "attribute",
"operation": "copy",
"priority": 20
},
{
"key": "ai.toolCall.name",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.tool.call.id",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "tool.id",
"context": "attribute",
"operation": "copy",
"priority": 20
},
{
"key": "ai.toolCall.id",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.tool.description",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "tool.description",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.tool.call.arguments",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "ai.toolCall.args",
"context": "attribute",
"operation": "copy",
"priority": 40
},
{
"key": "traceloop.entity.input",
"context": "attribute",
"operation": "copy",
"priority": 30
},
{
"key": "gcp.vertex.agent.tool_call_args",
"context": "attribute",
"operation": "copy",
"priority": 20
},
{
"key": "input.value",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
},
{
"name": "gen_ai.tool.call.result",
"fieldContext": "attribute",
"config": {
"sources": [
{
"key": "ai.toolCall.result",
"context": "attribute",
"operation": "copy",
"priority": 40
},
{
"key": "traceloop.entity.output",
"context": "attribute",
"operation": "copy",
"priority": 30
},
{
"key": "gcp.vertex.agent.tool_response",
"context": "attribute",
"operation": "copy",
"priority": 20
},
{
"key": "output.value",
"context": "attribute",
"operation": "copy",
"priority": 10
}
]
}
}
]
}
}

View File

@@ -69,10 +69,6 @@ func (h *handler) CreateGroup(rw http.ResponseWriter, r *http.Request) {
render.Error(rw, err)
return
}
if err := req.Validate(); err != nil {
render.Error(rw, err)
return
}
group := spantypes.NewSpanMapperGroup(orgID, claims.Email, req)
@@ -195,10 +191,6 @@ func (h *handler) CreateMapper(rw http.ResponseWriter, r *http.Request) {
render.Error(rw, err)
return
}
if err := req.Validate(); err != nil {
render.Error(rw, err)
return
}
mapper := spantypes.NewSpanMapper(groupID, claims.Email, req)
if err := h.module.CreateMapper(ctx, orgID, groupID, mapper); err != nil {

View File

@@ -5,7 +5,6 @@ import (
"encoding/json"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/modules/spanmapper"
"github.com/SigNoz/signoz/pkg/query-service/agentConf"
@@ -15,24 +14,13 @@ import (
"github.com/SigNoz/signoz/pkg/valuer"
)
// maxTestSpans bounds the input size: every test request boots a full
// in-memory collector pipeline and is reachable with viewer access.
const maxTestSpans = 100
type module struct {
store spantypes.SpanMapperStore
flagger flagger.Flagger
registry spantypes.SpanMapperGroupRegistry
settings factory.ScopedProviderSettings
store spantypes.SpanMapperStore
flagger flagger.Flagger
}
func NewModule(store spantypes.SpanMapperStore, flagger flagger.Flagger, registry spantypes.SpanMapperGroupRegistry, providerSettings factory.ProviderSettings) spanmapper.Module {
return &module{
store: store,
flagger: flagger,
registry: registry,
settings: factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/modules/spanmapper/implspanmapper"),
}
func NewModule(store spantypes.SpanMapperStore, flagger flagger.Flagger) spanmapper.Module {
return &module{store: store, flagger: flagger}
}
func (module *module) ListGroups(ctx context.Context, orgID valuer.UUID, q *spantypes.ListSpanMapperGroupsQuery) ([]*spantypes.SpanMapperGroup, error) {
@@ -44,9 +32,6 @@ func (module *module) GetGroup(ctx context.Context, orgID, id valuer.UUID) (*spa
}
func (module *module) CreateGroup(ctx context.Context, orgID valuer.UUID, group *spantypes.SpanMapperGroup) error {
if module.registry.IsReserved(group.Name) {
return errors.Newf(errors.TypeInvalidInput, spantypes.ErrCodeMappingGroupNameReserved, "group name %q is reserved for a default group", group.Name)
}
return module.store.CreateGroup(ctx, group)
}
@@ -55,14 +40,10 @@ func (module *module) UpdateGroup(ctx context.Context, orgID, id valuer.UUID, na
if err != nil {
return err
}
if name != nil && *name != group.Name && module.registry.IsReserved(*name) {
return errors.Newf(errors.TypeInvalidInput, spantypes.ErrCodeMappingGroupNameReserved, "group name %q is reserved for a default group", *name)
}
if err := group.Update(name, condition, enabled, updatedBy); err != nil {
return err
}
group.Update(name, condition, enabled, updatedBy)
if err := module.store.UpdateGroup(ctx, group); err != nil {
err = module.store.UpdateGroup(ctx, group)
if err != nil {
return err
}
agentConf.NotifyConfigUpdate(ctx)
@@ -70,16 +51,10 @@ func (module *module) UpdateGroup(ctx context.Context, orgID, id valuer.UUID, na
}
func (module *module) DeleteGroup(ctx context.Context, orgID, id valuer.UUID) error {
group, err := module.store.GetGroup(ctx, orgID, id)
err := module.store.DeleteGroup(ctx, orgID, id)
if err != nil {
return err
}
if err := group.ErrIfNotDeletable(); err != nil {
return err
}
if err := module.store.DeleteGroup(ctx, orgID, id); err != nil {
return err
}
agentConf.NotifyConfigUpdate(ctx)
return nil
}
@@ -106,13 +81,14 @@ func (module *module) CreateMapper(ctx context.Context, orgID, groupID valuer.UU
}
func (module *module) UpdateMapper(ctx context.Context, orgID, groupID, id valuer.UUID, fieldContext spantypes.FieldContext, config *spantypes.SpanMapperConfig, enabled *bool, updatedBy string) error {
if _, err := module.store.GetGroup(ctx, orgID, groupID); err != nil {
return err
}
mapper, err := module.store.GetMapper(ctx, orgID, groupID, id)
if err != nil {
return err
}
if err := mapper.Update(fieldContext, config, enabled, updatedBy); err != nil {
return err
}
mapper.Update(fieldContext, config, enabled, updatedBy)
err = module.store.UpdateMapper(ctx, mapper)
if err != nil {
return err
@@ -122,14 +98,7 @@ func (module *module) UpdateMapper(ctx context.Context, orgID, groupID, id value
}
func (module *module) DeleteMapper(ctx context.Context, orgID, groupID, id valuer.UUID) error {
mapper, err := module.store.GetMapper(ctx, orgID, groupID, id)
if err != nil {
return err
}
if err := mapper.ErrIfNotDeletable(); err != nil {
return err
}
err = module.store.DeleteMapper(ctx, orgID, groupID, id)
err := module.store.DeleteMapper(ctx, orgID, groupID, id)
if err != nil {
return err
}
@@ -137,6 +106,10 @@ func (module *module) DeleteMapper(ctx context.Context, orgID, groupID, id value
return nil
}
// maxTestSpans bounds the input size: every test request boots a full
// in-memory collector pipeline and is reachable with viewer access.
const maxTestSpans = 100
func (module *module) TestMappers(ctx context.Context, orgID valuer.UUID, spans []spantypes.SpanMapperTestSpan, groups []*spantypes.SpanMapperGroupWithMappers) ([]spantypes.SpanMapperTestSpan, []string, error) {
if len(spans) == 0 {
return nil, nil, errors.New(errors.TypeInvalidInput, spantypes.ErrCodeMappingInvalidInput, "'spans' must contain at least one span")
@@ -157,6 +130,37 @@ func (module *module) TestMappers(ctx context.Context, orgID valuer.UUID, spans
return out, collectorLogs, nil
}
// backfillMappers loads saved mappers for any enabled group whose Mappers is
// nil. Disabled groups are skipped: the simulation filters them out anyway,
// so there is no point loading their mappers or failing on their names.
func (module *module) backfillMappers(ctx context.Context, orgID valuer.UUID, groups []*spantypes.SpanMapperGroupWithMappers) ([]*spantypes.SpanMapperGroupWithMappers, error) {
savedGroups, err := module.store.ListGroups(ctx, orgID, nil)
if err != nil {
return nil, err
}
savedByName := make(map[string]*spantypes.SpanMapperGroup, len(savedGroups))
for _, g := range savedGroups {
savedByName[g.Name] = g
}
// For each group in the request, if Mappers is nil, load the saved mappers for that group name.
for _, g := range groups {
if g.Mappers != nil || !g.Group.Enabled {
continue
}
saved, ok := savedByName[g.Group.Name]
if !ok {
return nil, errors.Newf(errors.TypeNotFound, spantypes.ErrCodeMappingGroupNotFound, "no saved group named %q to load mappers from; send 'mappers' for new or edited groups", g.Group.Name)
}
loaded, err := module.store.ListMappers(ctx, orgID, saved.ID)
if err != nil {
return nil, err
}
g.Mappers = loaded
}
return groups, nil
}
func (module *module) AgentFeatureType() agentConf.AgentFeatureType {
return spantypes.SpanAttrMappingFeatureType
}
@@ -192,37 +196,6 @@ func (module *module) RecommendAgentConfig(orgID valuer.UUID, currentConfYaml []
return updatedConf, string(serialized), nil
}
// backfillMappers loads saved mappers for any enabled group whose Mappers is
// nil. Disabled groups are skipped: the simulation filters them out anyway,
// so there is no point loading their mappers or failing on their names.
func (module *module) backfillMappers(ctx context.Context, orgID valuer.UUID, groups []*spantypes.SpanMapperGroupWithMappers) ([]*spantypes.SpanMapperGroupWithMappers, error) {
savedGroups, err := module.store.ListGroups(ctx, orgID, nil)
if err != nil {
return nil, err
}
savedByName := make(map[string]*spantypes.SpanMapperGroup, len(savedGroups))
for _, g := range savedGroups {
savedByName[g.Name] = g
}
// For each group in the request, if Mappers is nil, load the saved mappers for that group name.
for _, g := range groups {
if g.Mappers != nil || !g.Group.Enabled {
continue
}
saved, ok := savedByName[g.Group.Name]
if !ok {
return nil, errors.Newf(errors.TypeNotFound, spantypes.ErrCodeMappingGroupNotFound, "no saved group named %q to load mappers from; send 'mappers' for new or edited groups", g.Group.Name)
}
loaded, err := module.store.ListMappers(ctx, orgID, saved.ID)
if err != nil {
return nil, err
}
g.Mappers = loaded
}
return groups, nil
}
// listEnabledGroupsWithMappers returns groups with their mappers.
func (module *module) listEnabledGroupsWithMappers(ctx context.Context, orgID valuer.UUID) ([]*spantypes.SpanMapperGroupWithMappers, error) {
enabled := true

View File

@@ -1,228 +0,0 @@
package implspanmapper
import (
"context"
"path/filepath"
"strconv"
"testing"
"time"
"github.com/SigNoz/signoz/pkg/factory/factorytest"
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/SigNoz/signoz/pkg/sqlstore/sqlitesqlstore"
"github.com/SigNoz/signoz/pkg/types/spantypes"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const testUser = "user@signoz.io"
func newTestSQLStore(t *testing.T) sqlstore.SQLStore {
t.Helper()
store, err := sqlitesqlstore.New(context.Background(), factorytest.NewSettings(), sqlstore.Config{
Provider: "sqlite",
Connection: sqlstore.ConnectionConfig{MaxOpenConns: 10},
Sqlite: sqlstore.SqliteConfig{
Path: filepath.Join(t.TempDir(), "test.db"),
Mode: "wal",
BusyTimeout: 5 * time.Second,
TransactionMode: "deferred",
},
})
require.NoError(t, err)
for _, model := range []any{
(*spantypes.StorableSpanMapperGroup)(nil),
(*spantypes.StorableSpanMapper)(nil),
} {
_, err := store.BunDB().NewCreateTable().Model(model).IfNotExists().Exec(context.Background())
require.NoError(t, err)
}
_, err = store.BunDB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS uq_span_mapper_group_org_name ON span_mapper_group (org_id, name)`)
require.NoError(t, err)
_, err = store.BunDB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS uq_span_mapper_group_name ON span_mapper (group_id, name)`)
require.NoError(t, err)
return store
}
func newTestModule(t *testing.T, sqlStore sqlstore.SQLStore, definitions ...spantypes.SpanMapperGroupDefinition) *module {
t.Helper()
registry, err := spantypes.NewSpanMapperGroupRegistry(definitions)
require.NoError(t, err)
return NewModule(NewStore(sqlStore), nil, registry, factorytest.NewSettings()).(*module)
}
func newTestDefinition(t *testing.T, version int, body string) spantypes.SpanMapperGroupDefinition {
t.Helper()
definition, err := spantypes.NewSpanMapperGroupDefinition([]byte(`{"version": ` + strconv.Itoa(version) + `, "definition": ` + body + `}`))
require.NoError(t, err)
return definition
}
// llmV1 ships two mappers; llmV2 renames a source, adds a mapper and drops one.
const llmV1 = `{
"name": "llm",
"condition": {"attributes": [{"value": "model"}], "resource": []},
"enabled": true,
"mappers": [
{"name": "gen_ai.request.model", "fieldContext": "attribute", "config": {"sources": [
{"key": "llm.model_name", "context": "attribute", "operation": "copy", "priority": 20},
{"key": "ai.model.id", "context": "attribute", "operation": "copy", "priority": 10}
]}},
{"name": "gen_ai.input.messages", "fieldContext": "attribute", "config": {"sources": [
{"key": "gen_ai.prompt", "context": "attribute", "operation": "copy", "priority": 10}
]}}
]
}`
const llmV2 = `{
"name": "llm",
"condition": {"attributes": [{"value": "model"}, {"value": "llm."}], "resource": []},
"enabled": true,
"mappers": [
{"name": "gen_ai.request.model", "fieldContext": "attribute", "config": {"sources": [
{"key": "llm.model_name", "context": "attribute", "operation": "copy", "priority": 20},
{"key": "langfuse.observation.model.name", "context": "attribute", "operation": "copy", "priority": 10}
]}},
{"name": "gen_ai.provider.name", "fieldContext": "attribute", "config": {"sources": [
{"key": "llm.vendor", "context": "attribute", "operation": "copy", "priority": 10}
]}}
]
}`
func findMapper(t *testing.T, mappers []*spantypes.SpanMapper, name string) *spantypes.SpanMapper {
t.Helper()
for _, m := range mappers {
if m.Name == name {
return m
}
}
require.Failf(t, "mapper not found", "no mapper named %q", name)
return nil
}
func findSource(t *testing.T, sources []spantypes.SpanMapperSource, key string, origin spantypes.SpanMapperOrigin) spantypes.SpanMapperSource {
t.Helper()
for _, s := range sources {
if s.Key == key && s.Origin == origin {
return s
}
}
require.Failf(t, "source not found", "no %s source with key %q", origin.StringValue(), key)
return spantypes.SpanMapperSource{}
}
func TestReconcileUpgradeKeepsTogglesAndUserItems(t *testing.T) {
ctx := context.Background()
orgID := valuer.GenerateUUID()
sqlStore := newTestSQLStore(t)
v1 := newTestModule(t, sqlStore, newTestDefinition(t, 1, llmV1))
require.NoError(t, v1.ReconcileSystemGroups(ctx, orgID))
group, err := v1.store.GetGroupByName(ctx, orgID, "llm")
require.NoError(t, err)
mappers, err := v1.ListMappers(ctx, orgID, group.ID)
require.NoError(t, err)
// Switch the shipped substring off and add a user one.
off := false
require.NoError(t, v1.UpdateGroup(ctx, orgID, group.ID, nil, &spantypes.SpanMapperGroupCondition{
Attributes: []spantypes.SpanMapperGroupConditionKey{
{Value: "model", Enabled: false, Origin: spantypes.SpanMapperOriginSystem},
{Value: "gen_ai.request.model", Enabled: true, Origin: spantypes.SpanMapperOriginUser},
},
Resource: []spantypes.SpanMapperGroupConditionKey{},
}, &off, testUser))
// Switch a shipped source off, add a user override, and switch the mapper off.
model := findMapper(t, mappers, "gen_ai.request.model")
require.NoError(t, v1.UpdateMapper(ctx, orgID, group.ID, model.ID, spantypes.FieldContext{}, &spantypes.SpanMapperConfig{Sources: []spantypes.SpanMapperSource{
{Key: "llm.model_name", Context: spantypes.FieldContextSpanAttribute, Operation: spantypes.SpanMapperOperationCopy, Priority: 20, Enabled: false, Origin: spantypes.SpanMapperOriginSystem},
{Key: "llm.model_name", Context: spantypes.FieldContextSpanAttribute, Operation: spantypes.SpanMapperOperationMove, Priority: 1, Enabled: true, Origin: spantypes.SpanMapperOriginUser},
}}, &off, testUser))
// Add a user source to the mapper v2 stops shipping, so it must survive.
messages := findMapper(t, mappers, "gen_ai.input.messages")
require.NoError(t, v1.UpdateMapper(ctx, orgID, group.ID, messages.ID, spantypes.FieldContext{}, &spantypes.SpanMapperConfig{Sources: []spantypes.SpanMapperSource{
{Key: "input.value", Context: spantypes.FieldContextSpanAttribute, Operation: spantypes.SpanMapperOperationCopy, Priority: 1, Enabled: true},
}}, nil, testUser))
// A user mapper in the shipped group.
require.NoError(t, v1.CreateMapper(ctx, orgID, group.ID, spantypes.NewSpanMapper(group.ID, testUser, &spantypes.PostableSpanMapper{
Name: "gen_ai.custom", FieldContext: spantypes.FieldContextSpanAttribute, Enabled: true,
Config: spantypes.SpanMapperConfig{Sources: []spantypes.SpanMapperSource{{Key: "custom", Context: spantypes.FieldContextSpanAttribute, Operation: spantypes.SpanMapperOperationCopy, Priority: 1, Enabled: true}}},
})))
v2 := newTestModule(t, sqlStore, newTestDefinition(t, 2, llmV2))
require.NoError(t, v2.ReconcileSystemGroups(ctx, orgID))
upgraded, err := v2.GetGroup(ctx, orgID, group.ID)
require.NoError(t, err)
assert.Equal(t, 2, upgraded.Version)
assert.False(t, upgraded.Enabled)
assert.Equal(t, spantypes.ProvisionerIdentity, upgraded.UpdatedBy)
assert.Equal(t, []spantypes.SpanMapperGroupConditionKey{
{Value: "model", Enabled: false, Origin: spantypes.SpanMapperOriginSystem},
{Value: "llm.", Enabled: true, Origin: spantypes.SpanMapperOriginSystem},
{Value: "gen_ai.request.model", Enabled: true, Origin: spantypes.SpanMapperOriginUser},
}, upgraded.Condition.Attributes)
mappers, err = v2.ListMappers(ctx, orgID, group.ID)
require.NoError(t, err)
require.Len(t, mappers, 4)
model = findMapper(t, mappers, "gen_ai.request.model")
assert.False(t, model.Enabled)
assert.Equal(t, spantypes.SpanMapperOriginSystem, model.Origin)
assert.False(t, findSource(t, model.Config.Sources, "llm.model_name", spantypes.SpanMapperOriginSystem).Enabled)
assert.True(t, findSource(t, model.Config.Sources, "langfuse.observation.model.name", spantypes.SpanMapperOriginSystem).Enabled)
assert.Equal(t, spantypes.SpanMapperOperationMove, findSource(t, model.Config.Sources, "llm.model_name", spantypes.SpanMapperOriginUser).Operation)
assert.Len(t, model.Config.Sources, 3)
messages = findMapper(t, mappers, "gen_ai.input.messages")
assert.Equal(t, spantypes.SpanMapperOriginUser, messages.Origin)
require.Len(t, messages.Config.Sources, 1)
assert.Equal(t, "input.value", messages.Config.Sources[0].Key)
assert.Equal(t, spantypes.SpanMapperOriginSystem, findMapper(t, mappers, "gen_ai.provider.name").Origin)
assert.Equal(t, spantypes.SpanMapperOriginUser, findMapper(t, mappers, "gen_ai.custom").Origin)
// Shipping v1 again drops provider.name outright (no user sources) and
// re-adopts the surviving user mapper input.messages as a shipped one.
v3 := newTestModule(t, sqlStore, newTestDefinition(t, 3, llmV1))
require.NoError(t, v3.ReconcileSystemGroups(ctx, orgID))
mappers, err = v3.ListMappers(ctx, orgID, group.ID)
require.NoError(t, err)
require.Len(t, mappers, 3)
for _, m := range mappers {
assert.NotEqual(t, "gen_ai.provider.name", m.Name)
}
messages = findMapper(t, mappers, "gen_ai.input.messages")
assert.Equal(t, spantypes.SpanMapperOriginSystem, messages.Origin)
assert.Len(t, messages.Config.Sources, 2)
}
func TestReconcileDoesNotDowngrade(t *testing.T) {
ctx := context.Background()
orgID := valuer.GenerateUUID()
sqlStore := newTestSQLStore(t)
require.NoError(t, newTestModule(t, sqlStore, newTestDefinition(t, 2, llmV2)).ReconcileSystemGroups(ctx, orgID))
older := newTestModule(t, sqlStore, newTestDefinition(t, 1, llmV1))
require.NoError(t, older.ReconcileSystemGroups(ctx, orgID))
group, err := older.store.GetGroupByName(ctx, orgID, "llm")
require.NoError(t, err)
assert.Equal(t, 2, group.Version)
mappers, err := older.ListMappers(ctx, orgID, group.ID)
require.NoError(t, err)
findMapper(t, mappers, "gen_ai.provider.name")
}

View File

@@ -1,187 +0,0 @@
package implspanmapper
import (
"context"
"log/slog"
"slices"
"time"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/query-service/agentConf"
"github.com/SigNoz/signoz/pkg/types/spantypes"
"github.com/SigNoz/signoz/pkg/valuer"
)
func (module *module) ReconcileSystemGroups(ctx context.Context, orgID valuer.UUID) error {
for _, definition := range module.registry.List() {
if err := module.reconcileSystemGroup(ctx, orgID, definition); err != nil {
return err
}
}
agentConf.NotifyConfigUpdate(ctx)
return nil
}
// reconcileSystemGroup brings one org's copy of a definition to the shipped
// version in a single transaction. A concurrent provisioner (another replica,
// or the org-creation hook racing the startup sweep) loses on the group's
// unique (org_id, name) index and is treated as a no-op.
func (module *module) reconcileSystemGroup(ctx context.Context, orgID valuer.UUID, definition spantypes.SpanMapperGroupDefinition) error {
err := module.store.RunInTx(ctx, func(ctx context.Context) error {
group, err := module.store.GetGroupByName(ctx, orgID, definition.Name())
if err != nil && errors.Ast(err, errors.TypeNotFound) {
group = newSystemGroup(orgID, definition)
err = module.store.CreateGroup(ctx, group)
}
if err != nil {
return err
}
if group.Origin != spantypes.SpanMapperOriginSystem {
module.settings.Logger().WarnContext(ctx, "skipping default span mapper group: a user group holds its name", slog.String("name", definition.Name()), slog.String("org_id", orgID.StringValue()))
return nil
}
if group.Version >= definition.Version {
return nil
}
return module.applyDefinition(ctx, orgID, group, definition)
})
if err != nil && errors.Ast(err, errors.TypeAlreadyExists) {
module.settings.Logger().DebugContext(ctx, "default span mapper group provisioned concurrently", slog.String("name", definition.Name()), slog.String("org_id", orgID.StringValue()))
return nil
}
return err
}
// applyDefinition replaces every shipped item with the definition, carrying each
// enabled flag over by identity, and leaves user items untouched. A mapper that
// is no longer shipped is deleted unless the user added sources to it, in which
// case it survives as a user mapper.
func (module *module) applyDefinition(ctx context.Context, orgID valuer.UUID, group *spantypes.SpanMapperGroup, definition spantypes.SpanMapperGroupDefinition) error {
mappers, err := module.store.ListMappers(ctx, orgID, group.ID)
if err != nil {
return err
}
byName := make(map[string]*spantypes.SpanMapper, len(mappers))
for _, m := range mappers {
byName[m.Name] = m
}
now := time.Now()
for i := range definition.Definition.Mappers {
pm := &definition.Definition.Mappers[i]
mapper, exists := byName[pm.Name]
delete(byName, pm.Name)
if !exists {
if err := module.store.CreateMapper(ctx, newSystemMapper(group.ID, pm)); err != nil {
return err
}
continue
}
mapper.Config.Sources = mergeShippedSources(mapper.Config.Sources, pm.Config.Sources)
mapper.FieldContext = pm.FieldContext
mapper.Origin = spantypes.SpanMapperOriginSystem
mapper.UpdatedAt = now
mapper.UpdatedBy = spantypes.ProvisionerIdentity
if err := module.store.UpdateMapper(ctx, mapper); err != nil {
return err
}
}
// Whatever is left in byName is not shipped any more.
for _, mapper := range byName {
if mapper.Origin != spantypes.SpanMapperOriginSystem {
continue
}
mapper.Config.Sources = mergeShippedSources(mapper.Config.Sources, nil)
if len(mapper.Config.Sources) == 0 {
if err := module.store.DeleteMapper(ctx, orgID, group.ID, mapper.ID); err != nil {
return err
}
continue
}
mapper.Origin = spantypes.SpanMapperOriginUser
mapper.UpdatedAt = now
mapper.UpdatedBy = spantypes.ProvisionerIdentity
if err := module.store.UpdateMapper(ctx, mapper); err != nil {
return err
}
}
shipped := definition.Definition.Condition
group.Condition = spantypes.SpanMapperGroupCondition{
Attributes: mergeShippedConditionKeys(group.Condition.Attributes, shipped.Attributes),
Resource: mergeShippedConditionKeys(group.Condition.Resource, shipped.Resource),
}
group.Version = definition.Version
group.UpdatedAt = now
group.UpdatedBy = spantypes.ProvisionerIdentity
if err := module.store.UpdateGroup(ctx, group); err != nil {
return err
}
module.settings.Logger().InfoContext(ctx, "applied default span mapper group", slog.String("name", definition.Name()), slog.Int("version", definition.Version), slog.String("org_id", orgID.StringValue()))
return nil
}
// newSystemGroup is the empty shell applyDefinition fills: version 0 so the
// definition is applied right after the row exists.
func newSystemGroup(orgID valuer.UUID, definition spantypes.SpanMapperGroupDefinition) *spantypes.SpanMapperGroup {
group := spantypes.NewSpanMapperGroup(orgID, spantypes.ProvisionerIdentity, &definition.Definition.PostableSpanMapperGroup)
group.Condition = definition.Definition.Condition
group.Enabled = true
group.Origin = spantypes.SpanMapperOriginSystem
return group
}
func newSystemMapper(groupID valuer.UUID, pm *spantypes.PostableSpanMapper) *spantypes.SpanMapper {
mapper := spantypes.NewSpanMapper(groupID, spantypes.ProvisionerIdentity, pm)
mapper.Config = pm.Config
mapper.Enabled = true
mapper.Origin = spantypes.SpanMapperOriginSystem
return mapper
}
// mergeShippedConditionKeys returns the shipped keys, each keeping the enabled
// flag of the stored system key with the same value, followed by the stored
// user keys.
func mergeShippedConditionKeys(stored, shipped []spantypes.SpanMapperGroupConditionKey) []spantypes.SpanMapperGroupConditionKey {
out := make([]spantypes.SpanMapperGroupConditionKey, 0, len(stored)+len(shipped))
for _, k := range shipped {
idx := slices.IndexFunc(stored, func(s spantypes.SpanMapperGroupConditionKey) bool {
return s.Origin == spantypes.SpanMapperOriginSystem && s.Value == k.Value
})
if idx != -1 {
k.Enabled = stored[idx].Enabled
}
out = append(out, k)
}
for _, k := range stored {
if k.Origin != spantypes.SpanMapperOriginSystem {
out = append(out, k)
}
}
return out
}
// mergeShippedSources returns the shipped sources, each keeping the enabled
// flag of the stored system source with the same key and context, followed by
// the stored user sources.
func mergeShippedSources(stored, shipped []spantypes.SpanMapperSource) []spantypes.SpanMapperSource {
out := make([]spantypes.SpanMapperSource, 0, len(stored)+len(shipped))
for _, s := range shipped {
idx := slices.IndexFunc(stored, func(o spantypes.SpanMapperSource) bool {
return o.Origin == spantypes.SpanMapperOriginSystem && o.Key == s.Key && o.Context == s.Context
})
if idx != -1 {
s.Enabled = stored[idx].Enabled
}
out = append(out, s)
}
for _, s := range stored {
if s.Origin != spantypes.SpanMapperOriginSystem {
out = append(out, s)
}
}
return out
}

View File

@@ -1,81 +0,0 @@
package implspanmapper
import (
"context"
"log/slog"
"time"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/modules/organization"
"github.com/SigNoz/signoz/pkg/modules/spanmapper"
)
const reconcileRetryInterval = 30 * time.Second
type service struct {
settings factory.ScopedProviderSettings
module spanmapper.Module
orgGetter organization.Getter
stopC chan struct{}
healthyC chan struct{}
}
// NewService reconciles every org's default mapping groups once at startup.
// Orgs created later are reconciled by the organization setter instead.
func NewService(providerSettings factory.ProviderSettings, module spanmapper.Module, orgGetter organization.Getter) factory.Service {
return &service{
settings: factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/modules/spanmapper/implspanmapper"),
module: module,
orgGetter: orgGetter,
stopC: make(chan struct{}),
healthyC: make(chan struct{}),
}
}
func (service *service) Start(ctx context.Context) error {
ticker := time.NewTicker(reconcileRetryInterval)
defer ticker.Stop()
for {
err := service.reconcile(ctx)
if err == nil {
close(service.healthyC)
<-service.stopC
return nil
}
service.settings.Logger().WarnContext(ctx, "default span mapper group reconciliation failed, retrying", errors.Attr(err))
select {
case <-service.stopC:
return nil
case <-ticker.C:
}
}
}
func (service *service) Healthy() <-chan struct{} {
return service.healthyC
}
func (service *service) Stop(_ context.Context) error {
close(service.stopC)
return nil
}
func (service *service) reconcile(ctx context.Context) error {
orgs, err := service.orgGetter.ListByOwnedKeyRange(ctx)
if err != nil {
return err
}
for _, org := range orgs {
if err := service.module.ReconcileSystemGroups(ctx, org.ID); err != nil {
return errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "couldn't reconcile default span mapper groups for org %s", org.ID.StringValue())
}
}
service.settings.Logger().InfoContext(ctx, "default span mapper group reconciliation completed", slog.Int("orgs", len(orgs)))
return nil
}

View File

@@ -17,10 +17,6 @@ func NewStore(sqlstore sqlstore.SQLStore) spantypes.SpanMapperStore {
return &store{sqlstore: sqlstore}
}
func (s *store) RunInTx(ctx context.Context, cb func(ctx context.Context) error) error {
return s.sqlstore.RunInTxCtx(ctx, nil, cb)
}
func (s *store) CreateGroup(ctx context.Context, group *spantypes.SpanMapperGroup) error {
storable := group.ToStorable()
_, err := s.sqlstore.
@@ -38,7 +34,7 @@ func (s *store) GetGroup(ctx context.Context, orgID, id valuer.UUID) (*spantypes
storable := new(spantypes.StorableSpanMapperGroup)
err := s.sqlstore.
BunDBCtx(ctx).
BunDB().
NewSelect().
Model(storable).
Where("org_id = ?", orgID).
@@ -50,27 +46,11 @@ func (s *store) GetGroup(ctx context.Context, orgID, id valuer.UUID) (*spantypes
return storable.ToSpanMapperGroup(), nil
}
func (s *store) GetGroupByName(ctx context.Context, orgID valuer.UUID, name string) (*spantypes.SpanMapperGroup, error) {
storable := new(spantypes.StorableSpanMapperGroup)
err := s.sqlstore.
BunDBCtx(ctx).
NewSelect().
Model(storable).
Where("org_id = ?", orgID).
Where("name = ?", name).
Scan(ctx)
if err != nil {
return nil, s.sqlstore.WrapNotFoundErrf(err, spantypes.ErrCodeMappingGroupNotFound, "span mapper group %q not found", name)
}
return storable.ToSpanMapperGroup(), nil
}
func (s *store) ListGroups(ctx context.Context, orgID valuer.UUID, q *spantypes.ListSpanMapperGroupsQuery) ([]*spantypes.SpanMapperGroup, error) {
storables := make([]*spantypes.StorableSpanMapperGroup, 0)
sel := s.sqlstore.
BunDBCtx(ctx).
BunDB().
NewSelect().
Model(&storables).
Where("org_id = ?", orgID)
@@ -111,35 +91,38 @@ func (s *store) UpdateGroup(ctx context.Context, group *spantypes.SpanMapperGrou
}
func (s *store) DeleteGroup(ctx context.Context, orgID, id valuer.UUID) error {
return s.RunInTx(ctx, func(ctx context.Context) error {
db := s.sqlstore.BunDBCtx(ctx)
tx, err := s.sqlstore.BunDBCtx(ctx).BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
// Cascade: remove mappers belonging to this group first.
if _, err := db.NewDelete().
Model((*spantypes.StorableSpanMapper)(nil)).
Where("group_id = ?", id).
Exec(ctx); err != nil {
return err
}
// Cascade: remove mappers belonging to this group first.
if _, err := tx.NewDelete().
Model((*spantypes.StorableSpanMapper)(nil)).
Where("group_id = ?", id).
Exec(ctx); err != nil {
return err
}
res, err := db.NewDelete().
Model((*spantypes.StorableSpanMapperGroup)(nil)).
Where("org_id = ?", orgID).
Where("id = ?", id).
Exec(ctx)
if err != nil {
return err
}
res, err := tx.NewDelete().
Model((*spantypes.StorableSpanMapperGroup)(nil)).
Where("org_id = ?", orgID).
Where("id = ?", id).
Exec(ctx)
if err != nil {
return err
}
rowsAffected, err := res.RowsAffected()
if err != nil {
return err
}
if rowsAffected == 0 {
return errors.Newf(errors.TypeNotFound, spantypes.ErrCodeMappingGroupNotFound, "span mapper group %s not found", id)
}
return nil
})
rowsAffected, err := res.RowsAffected()
if err != nil {
return err
}
if rowsAffected == 0 {
return errors.Newf(errors.TypeNotFound, spantypes.ErrCodeMappingGroupNotFound, "span mapper group %s not found", id)
}
return tx.Commit()
}
func (s *store) CreateMapper(ctx context.Context, mapper *spantypes.SpanMapper) error {
@@ -163,7 +146,7 @@ func (s *store) GetMapper(ctx context.Context, orgID, groupID, id valuer.UUID) (
storable := new(spantypes.StorableSpanMapper)
err := s.sqlstore.
BunDBCtx(ctx).
BunDB().
NewSelect().
Model(storable).
Where("group_id = ?", groupID).
@@ -183,7 +166,7 @@ func (s *store) ListMappers(ctx context.Context, orgID, groupID valuer.UUID) ([]
storables := make([]*spantypes.StorableSpanMapper, 0)
if err := s.sqlstore.
BunDBCtx(ctx).
BunDB().
NewSelect().
Model(&storables).
Where("group_id = ?", groupID).

View File

@@ -28,10 +28,6 @@ type Module interface {
UpdateMapper(ctx context.Context, orgID, groupID, id valuer.UUID, fieldContext spantypes.FieldContext, config *spantypes.SpanMapperConfig, enabled *bool, updatedBy string) error
DeleteMapper(ctx context.Context, orgID, groupID, id valuer.UUID) error
TestMappers(ctx context.Context, orgID valuer.UUID, spans []spantypes.SpanMapperTestSpan, groups []*spantypes.SpanMapperGroupWithMappers) ([]spantypes.SpanMapperTestSpan, []string, error)
// ReconcileSystemGroups provisions or upgrades the shipped mapping groups
// for one org. It runs at startup for every org and again on org creation.
ReconcileSystemGroups(ctx context.Context, orgID valuer.UUID) error
}
// Handler defines the HTTP handler interface for mapping group and mapper endpoints.

View File

@@ -0,0 +1,516 @@
// Package sqlcompiler compiles list-page filter queries to relational-store WHERE clauses; telemetry queries stay on querybuilder's ClickHouse visitor.
package sqlcompiler
import (
"fmt"
"strings"
"time"
"github.com/SigNoz/signoz/pkg/parser/filterquery"
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
"github.com/SigNoz/signoz/pkg/sqlstore"
qbtypesv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/antlr4-go/antlr/v4"
sqlbuilder "github.com/huandu/go-sqlbuilder"
)
// bunPlaceholderFlavor is any flavor that renders the `?` placeholders bun expects.
const bunPlaceholderFlavor = sqlbuilder.SQLite
// FieldResolver is the per-feature policy: which keys exist and what each maps to.
type FieldResolver interface {
// ResolveComparison builds the predicate for one `key OP value` term; key keeps the user's casing.
ResolveComparison(v *Visitor, key string, operation qbtypesv5.FilterOperator, ctx *grammar.ComparisonContext) string
// FreeText builds the predicate for a bare token.
FreeText(v *Visitor, value string) string
}
// Compiled is a `?`-placeholder WHERE clause with its bun bind args.
type Compiled struct {
SQL string
Args []any
}
func (c Compiled) IsEmpty() bool {
return c.SQL == ""
}
// Compile on success returns a non-nil *Compiled, empty for an empty query; callers gate on IsEmpty, not nil.
func Compile(query string, formatter sqlstore.SQLFormatter, resolver FieldResolver) (*Compiled, []string) {
if len(strings.TrimSpace(query)) == 0 {
return &Compiled{}, nil
}
v := &Visitor{
sb: sqlbuilder.NewSelectBuilder(),
fmter: formatter,
resolver: resolver,
}
tree, _, collector := filterquery.Parse(query)
if len(collector.Errors) > 0 {
return nil, collector.Errors
}
condition, _ := v.visit(tree).(string)
if len(v.errors) > 0 {
return nil, v.errors
}
if condition == "" {
return &Compiled{}, nil
}
sql, arguments := v.sb.Args.CompileWithFlavor(condition, bunPlaceholderFlavor)
return &Compiled{SQL: sql, Args: arguments}, nil
}
// Visitor walks the parse tree and carries the per-compile SQL state; the
// FieldResolver calls its helpers back to build each predicate.
type Visitor struct {
grammar.BaseFilterQueryVisitor
sb *sqlbuilder.SelectBuilder
fmter sqlstore.SQLFormatter
errors []string
resolver FieldResolver
}
func (v *Visitor) SelectBuilder() *sqlbuilder.SelectBuilder {
return v.sb
}
func (v *Visitor) Formatter() sqlstore.SQLFormatter {
return v.fmter
}
func (v *Visitor) AddError(format string, arguments ...any) {
v.errors = append(v.errors, fmt.Sprintf(format, arguments...))
}
// StringOperation interns placeholders into sb so nested subquery arguments thread correctly.
func (v *Visitor) StringOperation(sb *sqlbuilder.SelectBuilder, ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression, keyForError string) string {
switch operation {
case qbtypesv5.FilterOperatorEqual:
val, ok := v.ExtractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
return sb.Equal(columnExpression, val)
case qbtypesv5.FilterOperatorNotEqual:
val, ok := v.ExtractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
return sb.NotEqual(columnExpression, val)
case qbtypesv5.FilterOperatorLike, qbtypesv5.FilterOperatorNotLike:
val, ok := v.ExtractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
if endsWithDanglingEscape(val) {
v.AddError("LIKE pattern for %q must not end with an unescaped backslash, use \\\\ to match a literal backslash", keyForError)
return ""
}
like := "LIKE"
if operation == qbtypesv5.FilterOperatorNotLike {
like = "NOT LIKE"
}
// ESCAPE pins backslash as the escape char (the Postgres default, SQLite has none).
return fmt.Sprintf("%s %s %s ESCAPE '\\'", columnExpression, like, sb.Var(val))
case qbtypesv5.FilterOperatorILike, qbtypesv5.FilterOperatorNotILike:
val, ok := v.ExtractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
if endsWithDanglingEscape(val) {
v.AddError("ILIKE pattern for %q must not end with an unescaped backslash, use \\\\ to match a literal backslash", keyForError)
return ""
}
// SQLite has no ILIKE and Postgres LIKE is case-sensitive, so LOWER both sides.
lowerColumn := string(v.fmter.LowerExpression(columnExpression))
like := "LIKE"
if operation == qbtypesv5.FilterOperatorNotILike {
like = "NOT LIKE"
}
return fmt.Sprintf("%s %s LOWER(%s) ESCAPE '\\'", lowerColumn, like, sb.Var(val))
case qbtypesv5.FilterOperatorContains, qbtypesv5.FilterOperatorNotContains:
val, ok := v.ExtractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
like := "LIKE"
if operation == qbtypesv5.FilterOperatorNotContains {
like = "NOT LIKE"
}
// Escape the user's % and _ so they match literally, then wrap in wildcards.
escaped := v.fmter.EscapeLikePattern(val)
return fmt.Sprintf("%s %s %s ESCAPE '\\'", columnExpression, like, sb.Var(fmt.Sprintf("%%%s%%", escaped)))
case qbtypesv5.FilterOperatorRegexp, qbtypesv5.FilterOperatorNotRegexp:
v.AddError("REGEXP filtering on %q is not supported", keyForError)
return ""
case qbtypesv5.FilterOperatorIn, qbtypesv5.FilterOperatorNotIn:
values, ok := v.ExtractStringValueList(ctx, keyForError)
if !ok {
return ""
}
arguments := make([]any, len(values))
for i, s := range values {
arguments[i] = s
}
if operation == qbtypesv5.FilterOperatorNotIn {
return sb.NotIn(columnExpression, arguments...)
}
return sb.In(columnExpression, arguments...)
}
v.AddError("operator %s on %q is not implemented", OperationName(operation), keyForError)
return ""
}
func (v *Visitor) TimestampComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression string) string {
switch operation {
case qbtypesv5.FilterOperatorEqual, qbtypesv5.FilterOperatorNotEqual,
qbtypesv5.FilterOperatorLessThan, qbtypesv5.FilterOperatorLessThanOrEq,
qbtypesv5.FilterOperatorGreaterThan, qbtypesv5.FilterOperatorGreaterThanOrEq:
t, ok := v.extractSingleTimestampValue(ctx)
if !ok {
return ""
}
switch operation {
case qbtypesv5.FilterOperatorEqual:
return v.sb.Equal(columnExpression, t)
case qbtypesv5.FilterOperatorNotEqual:
return v.sb.NotEqual(columnExpression, t)
case qbtypesv5.FilterOperatorLessThan:
return v.sb.LessThan(columnExpression, t)
case qbtypesv5.FilterOperatorLessThanOrEq:
return v.sb.LessEqualThan(columnExpression, t)
case qbtypesv5.FilterOperatorGreaterThan:
return v.sb.GreaterThan(columnExpression, t)
case qbtypesv5.FilterOperatorGreaterThanOrEq:
return v.sb.GreaterEqualThan(columnExpression, t)
}
case qbtypesv5.FilterOperatorBetween, qbtypesv5.FilterOperatorNotBetween:
timestamps, ok := v.extractTwoTimestampValues(ctx)
if !ok {
return ""
}
if operation == qbtypesv5.FilterOperatorNotBetween {
return v.sb.NotBetween(columnExpression, timestamps[0], timestamps[1])
}
return v.sb.Between(columnExpression, timestamps[0], timestamps[1])
}
v.AddError("operator %s on timestamp is not implemented", OperationName(operation))
return ""
}
func (v *Visitor) BoolComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression string) string {
value, ok := v.extractSingleBoolValue(ctx)
if !ok {
return ""
}
if operation == qbtypesv5.FilterOperatorNotEqual {
return v.sb.NotEqual(columnExpression, value)
}
return v.sb.Equal(columnExpression, value)
}
// A pattern ending in an unescaped backslash never matches on sqlite and errors on Postgres.
func endsWithDanglingEscape(value string) bool {
trailing := len(value) - len(strings.TrimRight(value, `\`))
return trailing%2 == 1
}
// FreeTextContains COALESCEs the column so NOT (...) does not go NULL and drop rows where it is absent.
func (v *Visitor) FreeTextContains(sb *sqlbuilder.SelectBuilder, columnExpression, value string) string {
lowerColumn := string(v.fmter.LowerExpression(fmt.Sprintf("COALESCE(%s, '')", columnExpression)))
pattern := fmt.Sprintf("%%%s%%", v.fmter.EscapeLikePattern(value))
return fmt.Sprintf("%s LIKE LOWER(%s) ESCAPE '\\'", lowerColumn, sb.Var(pattern))
}
func (v *Visitor) ExtractSingleStringValue(ctx *grammar.ComparisonContext, keyForError string) (string, bool) {
values := ctx.AllValue()
if len(values) != 1 {
v.AddError("expected exactly one value for %q", keyForError)
return "", false
}
return v.extractStringValue(values[0], keyForError)
}
func (v *Visitor) ExtractStringValueList(ctx *grammar.ComparisonContext, keyForError string) ([]string, bool) {
var valuesCtx []grammar.IValueContext
switch {
case ctx.InClause() != nil:
inClause := ctx.InClause()
if inClause.ValueList() != nil {
valuesCtx = inClause.ValueList().AllValue()
} else {
valuesCtx = []grammar.IValueContext{inClause.Value()}
}
case ctx.NotInClause() != nil:
notInClause := ctx.NotInClause()
if notInClause.ValueList() != nil {
valuesCtx = notInClause.ValueList().AllValue()
} else {
valuesCtx = []grammar.IValueContext{notInClause.Value()}
}
default:
v.AddError("IN clause is missing for %q", keyForError)
return nil, false
}
if len(valuesCtx) == 0 {
v.AddError("IN list for %q is empty", keyForError)
return nil, false
}
out := make([]string, 0, len(valuesCtx))
for _, valueContext := range valuesCtx {
s, ok := v.extractStringValue(valueContext, keyForError)
if !ok {
return nil, false
}
out = append(out, s)
}
return out, true
}
func (v *Visitor) extractSingleBoolValue(ctx *grammar.ComparisonContext) (bool, bool) {
values := ctx.AllValue()
if len(values) != 1 {
v.AddError("expected a single boolean (true/false)")
return false, false
}
return v.extractBoolValue(values[0])
}
func (v *Visitor) extractSingleTimestampValue(ctx *grammar.ComparisonContext) (time.Time, bool) {
values := ctx.AllValue()
if len(values) != 1 {
v.AddError("expected a single RFC3339 timestamp")
return time.Time{}, false
}
return v.extractTimestampValue(values[0])
}
func (v *Visitor) extractTwoTimestampValues(ctx *grammar.ComparisonContext) ([2]time.Time, bool) {
values := ctx.AllValue()
if len(values) != 2 {
v.AddError("BETWEEN expects two RFC3339 timestamps")
return [2]time.Time{}, false
}
first, ok1 := v.extractTimestampValue(values[0])
second, ok2 := v.extractTimestampValue(values[1])
if !ok1 || !ok2 {
return [2]time.Time{}, false
}
return [2]time.Time{first, second}, true
}
func (v *Visitor) extractStringValue(ctx grammar.IValueContext, keyForError string) (string, bool) {
if ctx.QUOTED_TEXT() != nil {
return trimQuotes(ctx.QUOTED_TEXT().GetText()), true
}
if ctx.KEY() != nil {
return ctx.KEY().GetText(), true
}
v.AddError("expected a string value for %q, got %q", keyForError, ctx.GetText())
return "", false
}
func (v *Visitor) extractBoolValue(ctx grammar.IValueContext) (bool, bool) {
if ctx.BOOL() == nil {
v.AddError("expected a boolean (true/false), got %q", ctx.GetText())
return false, false
}
return strings.EqualFold(ctx.BOOL().GetText(), "true"), true
}
func (v *Visitor) extractTimestampValue(ctx grammar.IValueContext) (time.Time, bool) {
if ctx.QUOTED_TEXT() == nil {
v.AddError("expected an RFC3339 timestamp string, got %q", ctx.GetText())
return time.Time{}, false
}
raw := trimQuotes(ctx.QUOTED_TEXT().GetText())
t, err := time.Parse(time.RFC3339, raw)
if err != nil {
v.AddError("invalid RFC3339 timestamp %q: %s", raw, err.Error())
return time.Time{}, false
}
return t, true
}
func (v *Visitor) visit(tree antlr.ParseTree) any {
if tree == nil {
return nil
}
return tree.Accept(v)
}
func (v *Visitor) VisitQuery(ctx *grammar.QueryContext) any {
return v.visit(ctx.Expression())
}
func (v *Visitor) VisitExpression(ctx *grammar.ExpressionContext) any {
return v.visit(ctx.OrExpression())
}
func (v *Visitor) VisitOrExpression(ctx *grammar.OrExpressionContext) any {
parts := ctx.AllAndExpression()
conditions := make([]string, 0, len(parts))
for _, part := range parts {
if condition, ok := v.visit(part).(string); ok && condition != "" {
conditions = append(conditions, condition)
}
}
switch len(conditions) {
case 0:
return ""
case 1:
return conditions[0]
default:
return v.sb.Or(conditions...)
}
}
func (v *Visitor) VisitAndExpression(ctx *grammar.AndExpressionContext) any {
parts := ctx.AllUnaryExpression()
conditions := make([]string, 0, len(parts))
for _, part := range parts {
if condition, ok := v.visit(part).(string); ok && condition != "" {
conditions = append(conditions, condition)
}
}
switch len(conditions) {
case 0:
return ""
case 1:
return conditions[0]
default:
return v.sb.And(conditions...)
}
}
func (v *Visitor) VisitUnaryExpression(ctx *grammar.UnaryExpressionContext) any {
condition, _ := v.visit(ctx.Primary()).(string)
if condition == "" {
return ""
}
if ctx.NOT() != nil {
return fmt.Sprintf("NOT (%s)", condition)
}
return condition
}
func (v *Visitor) VisitPrimary(ctx *grammar.PrimaryContext) any {
if ctx.OrExpression() != nil {
return v.visit(ctx.OrExpression())
}
if ctx.Comparison() != nil {
return v.visit(ctx.Comparison())
}
// A quoted lone token matches its contents literally, the escape hatch for a phrase that looks like DSL.
return v.resolver.FreeText(v, trimQuotes(ctx.GetText()))
}
func (v *Visitor) VisitComparison(ctx *grammar.ComparisonContext) any {
key := strings.TrimSpace(ctx.Key().GetText())
operation, ok := v.extractOperation(ctx)
if !ok {
return ""
}
return v.resolver.ResolveComparison(v, key, operation, ctx)
}
func (v *Visitor) extractOperation(ctx *grammar.ComparisonContext) (qbtypesv5.FilterOperator, bool) {
maybeNot := func(operation qbtypesv5.FilterOperator) qbtypesv5.FilterOperator {
if ctx.NOT() != nil {
return operation.Inverse()
}
return operation
}
switch {
case ctx.EQUALS() != nil:
return qbtypesv5.FilterOperatorEqual, true
case ctx.NOT_EQUALS() != nil, ctx.NEQ() != nil:
return qbtypesv5.FilterOperatorNotEqual, true
case ctx.LT() != nil:
return qbtypesv5.FilterOperatorLessThan, true
case ctx.LE() != nil:
return qbtypesv5.FilterOperatorLessThanOrEq, true
case ctx.GT() != nil:
return qbtypesv5.FilterOperatorGreaterThan, true
case ctx.GE() != nil:
return qbtypesv5.FilterOperatorGreaterThanOrEq, true
case ctx.BETWEEN() != nil:
return maybeNot(qbtypesv5.FilterOperatorBetween), true
case ctx.LIKE() != nil:
return maybeNot(qbtypesv5.FilterOperatorLike), true
case ctx.ILIKE() != nil:
return maybeNot(qbtypesv5.FilterOperatorILike), true
case ctx.CONTAINS() != nil:
return maybeNot(qbtypesv5.FilterOperatorContains), true
case ctx.REGEXP() != nil:
return maybeNot(qbtypesv5.FilterOperatorRegexp), true
case ctx.InClause() != nil:
return qbtypesv5.FilterOperatorIn, true
case ctx.NotInClause() != nil:
return qbtypesv5.FilterOperatorNotIn, true
case ctx.EXISTS() != nil:
return maybeNot(qbtypesv5.FilterOperatorExists), true
}
v.AddError("could not determine operator in expression %q", ctx.GetText())
return qbtypesv5.FilterOperatorUnknown, false
}
// OperationName is the user-facing spelling, used only in error messages.
func OperationName(operation qbtypesv5.FilterOperator) string {
switch operation {
case qbtypesv5.FilterOperatorEqual:
return "="
case qbtypesv5.FilterOperatorNotEqual:
return "!="
case qbtypesv5.FilterOperatorLessThan:
return "<"
case qbtypesv5.FilterOperatorLessThanOrEq:
return "<="
case qbtypesv5.FilterOperatorGreaterThan:
return ">"
case qbtypesv5.FilterOperatorGreaterThanOrEq:
return ">="
case qbtypesv5.FilterOperatorBetween:
return "BETWEEN"
case qbtypesv5.FilterOperatorNotBetween:
return "NOT BETWEEN"
case qbtypesv5.FilterOperatorLike:
return "LIKE"
case qbtypesv5.FilterOperatorNotLike:
return "NOT LIKE"
case qbtypesv5.FilterOperatorILike:
return "ILIKE"
case qbtypesv5.FilterOperatorNotILike:
return "NOT ILIKE"
case qbtypesv5.FilterOperatorContains:
return "CONTAINS"
case qbtypesv5.FilterOperatorNotContains:
return "NOT CONTAINS"
case qbtypesv5.FilterOperatorRegexp:
return "REGEXP"
case qbtypesv5.FilterOperatorNotRegexp:
return "NOT REGEXP"
case qbtypesv5.FilterOperatorIn:
return "IN"
case qbtypesv5.FilterOperatorNotIn:
return "NOT IN"
case qbtypesv5.FilterOperatorExists:
return "EXISTS"
case qbtypesv5.FilterOperatorNotExists:
return "NOT EXISTS"
}
return "?"
}
func trimQuotes(s string) string {
if len(s) >= 2 {
if (s[0] == '"' && s[len(s)-1] == '"') || (s[0] == '\'' && s[len(s)-1] == '\'') {
s = s[1 : len(s)-1]
}
}
s = strings.ReplaceAll(s, `\\`, `\`)
s = strings.ReplaceAll(s, `\'`, `'`)
return s
}

View File

@@ -18,7 +18,6 @@ import (
"github.com/SigNoz/signoz/pkg/modules/dashboard/impldashboard"
"github.com/SigNoz/signoz/pkg/modules/organization/implorganization"
"github.com/SigNoz/signoz/pkg/modules/retention/implretention"
"github.com/SigNoz/signoz/pkg/modules/spanmapper/implspanmapper"
"github.com/SigNoz/signoz/pkg/modules/tag/impltag"
"github.com/SigNoz/signoz/pkg/modules/user/impluser"
"github.com/SigNoz/signoz/pkg/querier"
@@ -62,10 +61,7 @@ func TestNewHandlers(t *testing.T) {
userGetter := impluser.NewGetter(impluser.NewStore(sqlstore, providerSettings), userRoleStore, flagger)
retentionGetter := implretention.NewGetter(implretention.NewStore(sqlstore))
spanMapperRegistry, err := implspanmapper.NewSystemGroupRegistry()
require.NoError(t, err)
spanMapperModule := implspanmapper.NewModule(implspanmapper.NewStore(sqlstore), flagger, spanMapperRegistry, providerSettings)
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, nil, nil, nil, nil, nil, nil, nil, queryParser, Config{}, dashboardModule, userGetter, userRoleStore, nil, nil, nil, retentionGetter, flagger, tagModule, nil, spanMapperModule)
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, nil, nil, nil, nil, nil, nil, nil, queryParser, Config{}, dashboardModule, userGetter, userRoleStore, nil, nil, nil, retentionGetter, flagger, tagModule, nil)
querierHandler := querier.NewHandler(providerSettings, nil, nil)
registryHandler := factory.NewHandler(nil)

View File

@@ -45,6 +45,7 @@ import (
"github.com/SigNoz/signoz/pkg/modules/session"
"github.com/SigNoz/signoz/pkg/modules/session/implsession"
"github.com/SigNoz/signoz/pkg/modules/spanmapper"
"github.com/SigNoz/signoz/pkg/modules/spanmapper/implspanmapper"
"github.com/SigNoz/signoz/pkg/modules/spanpercentile"
"github.com/SigNoz/signoz/pkg/modules/spanpercentile/implspanpercentile"
"github.com/SigNoz/signoz/pkg/modules/tag"
@@ -123,10 +124,9 @@ func NewModules(
fl flagger.Flagger,
tagModule tag.Module,
metricReductionRule metricreductionrule.Module,
spanMapper spanmapper.Module,
) Modules {
quickfilter := implquickfilter.NewModule(implquickfilter.NewStore(sqlstore))
orgSetter := implorganization.NewSetter(implorganization.NewStore(sqlstore), alertmanager, quickfilter, dashboard, spanMapper)
orgSetter := implorganization.NewSetter(implorganization.NewStore(sqlstore), alertmanager, quickfilter, dashboard)
// Cleanup callbacks from other modules, invoked when a user is deleted.
onDeleteUser := []user.OnDeleteUser{
dashboard.DeletePreferencesForUser,
@@ -162,7 +162,7 @@ func NewModules(
RuleStateHistory: implrulestatehistory.NewModule(implrulestatehistory.NewStore(telemetryStore, telemetryMetadataStore, providerSettings.Logger), ruleStore),
CloudIntegration: cloudIntegrationModule,
TraceDetail: impltracedetail.NewModule(impltracedetail.NewTraceStore(telemetryStore), providerSettings, config.TraceDetail),
SpanMapper: spanMapper,
SpanMapper: implspanmapper.NewModule(implspanmapper.NewStore(sqlstore), fl),
LLMPricingRule: impllmpricingrule.NewModule(impllmpricingrule.NewStore(sqlstore), fl, querier),
Tag: tagModule,
}

View File

@@ -21,7 +21,6 @@ import (
"github.com/SigNoz/signoz/pkg/modules/retention/implretention"
"github.com/SigNoz/signoz/pkg/modules/serviceaccount"
"github.com/SigNoz/signoz/pkg/modules/serviceaccount/implserviceaccount"
"github.com/SigNoz/signoz/pkg/modules/spanmapper/implspanmapper"
"github.com/SigNoz/signoz/pkg/modules/tag/impltag"
"github.com/SigNoz/signoz/pkg/modules/user/impluser"
"github.com/SigNoz/signoz/pkg/queryparser"
@@ -69,11 +68,7 @@ func TestNewModules(t *testing.T) {
retentionGetter := implretention.NewGetter(implretention.NewStore(sqlstore))
spanMapperRegistry, err := implspanmapper.NewSystemGroupRegistry()
require.NoError(t, err)
spanMapperModule := implspanmapper.NewModule(implspanmapper.NewStore(sqlstore), flagger, spanMapperRegistry, providerSettings)
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, nil, nil, nil, nil, nil, nil, nil, queryParser, Config{}, dashboardModule, userGetter, userRoleStore, serviceAccount, serviceAccountGetter, implcloudintegration.NewModule(), retentionGetter, flagger, tagModule, implmetricreductionrule.NewModule(), spanMapperModule)
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, nil, nil, nil, nil, nil, nil, nil, queryParser, Config{}, dashboardModule, userGetter, userRoleStore, serviceAccount, serviceAccountGetter, implcloudintegration.NewModule(), retentionGetter, flagger, tagModule, implmetricreductionrule.NewModule())
reflectVal := reflect.ValueOf(modules)
for i := 0; i < reflectVal.NumField(); i++ {

View File

@@ -254,7 +254,6 @@ func NewSQLMigrationProviderFactories(
sqlmigration.NewAddIngestionTuplesFactory(sqlstore),
sqlmigration.NewAddSubscriptionTuplesFactory(sqlstore),
sqlmigration.NewNormalizeQuickFilterFieldsFactory(sqlstore),
sqlmigration.NewAddSpanMapperOriginFactory(sqlstore, sqlschema),
)
}

View File

@@ -36,7 +36,6 @@ import (
"github.com/SigNoz/signoz/pkg/modules/rulestatehistory"
"github.com/SigNoz/signoz/pkg/modules/serviceaccount"
"github.com/SigNoz/signoz/pkg/modules/serviceaccount/implserviceaccount"
"github.com/SigNoz/signoz/pkg/modules/spanmapper/implspanmapper"
"github.com/SigNoz/signoz/pkg/modules/tag"
"github.com/SigNoz/signoz/pkg/modules/tag/impltag"
"github.com/SigNoz/signoz/pkg/modules/user/impluser"
@@ -550,15 +549,7 @@ func New(
metricReductionRuleModule := metricReductionRuleModuleCallback(sqlstore, telemetrystore, dashboard, queryParser, licensing, flagger, telemetryMetadataStore, providerSettings, config.MetricsExplorer.TelemetryStore.Threads)
// Initialize all modules
// The default mapping group registry is parsed here so a malformed embedded
// definition fails startup instead of a request.
spanMapperRegistry, err := implspanmapper.NewSystemGroupRegistry()
if err != nil {
return nil, err
}
spanMapperModule := implspanmapper.NewModule(implspanmapper.NewStore(sqlstore), flagger, spanMapperRegistry, providerSettings)
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, analytics, querier, telemetrystore, telemetryMetadataStore, authNs, authz, cache, queryParser, config, dashboard, userGetter, userRoleStore, serviceAccount, serviceAccountGetter, cloudIntegrationModule, retentionGetter, flagger, tagModule, metricReductionRuleModule, spanMapperModule)
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, analytics, querier, telemetrystore, telemetryMetadataStore, authNs, authz, cache, queryParser, config, dashboard, userGetter, userRoleStore, serviceAccount, serviceAccountGetter, cloudIntegrationModule, retentionGetter, flagger, tagModule, metricReductionRuleModule)
// Initialize ruler from the variant-specific provider factories
rulerInstance, err := factory.NewProviderFromNamedMap(ctx, providerSettings, config.Ruler, rulerProviderFactories(cache, alertmanager, sqlstore, telemetrystore, telemetryMetadataStore, prometheus, orgGetter, modules.RuleStateHistory, querier, queryParser), "signoz")
@@ -628,7 +619,6 @@ func New(
factory.NewNamedService(factory.MustNewName("meterreporter"), meterReporter, factory.MustNewName("licensing")),
factory.NewNamedService(factory.MustNewName("ruler"), rulerInstance),
factory.NewNamedService(factory.MustNewName("systemdashboard"), impldashboard.NewService(providerSettings, dashboard, orgGetter)),
factory.NewNamedService(factory.MustNewName("spanmappergroup"), implspanmapper.NewService(providerSettings, spanMapperModule, orgGetter)),
)
if err != nil {
return nil, err

View File

@@ -1,93 +0,0 @@
package sqlmigration
import (
"context"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/sqlschema"
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/uptrace/bun"
"github.com/uptrace/bun/migrate"
)
type addSpanMapperOrigin struct {
sqlstore sqlstore.SQLStore
sqlschema sqlschema.SQLSchema
}
func NewAddSpanMapperOriginFactory(sqlstore sqlstore.SQLStore, sqlschema sqlschema.SQLSchema) factory.ProviderFactory[SQLMigration, Config] {
return factory.NewProviderFactory(
factory.MustNewName("add_span_mapper_origin"),
func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
return &addSpanMapperOrigin{sqlstore: sqlstore, sqlschema: sqlschema}, nil
},
)
}
func (migration *addSpanMapperOrigin) Register(migrations *migrate.Migrations) error {
return migrations.Register(migration.Up, migration.Down)
}
// Up adds the ownership columns that let SigNoz ship default mapping groups
// alongside user ones.
func (migration *addSpanMapperOrigin) Up(ctx context.Context, db *bun.DB) error {
// span_mapper references span_mapper_group and both have foreign keys, so
// enforcement must be off for the SQLite recreate-table fallback.
if err := migration.sqlschema.ToggleFKEnforcement(ctx, db, false); err != nil {
return err
}
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
groupTable, groupUniqueConstraints, err := migration.sqlschema.GetTable(ctx, sqlschema.TableName("span_mapper_group"))
if err != nil {
return err
}
sqls := migration.sqlschema.Operator().AddColumn(groupTable, groupUniqueConstraints, &sqlschema.Column{
Name: sqlschema.ColumnName("origin"),
DataType: sqlschema.DataTypeText,
Nullable: false,
Default: "'user'",
}, "user")
sqls = append(sqls, migration.sqlschema.Operator().AddColumn(groupTable, groupUniqueConstraints, &sqlschema.Column{
Name: sqlschema.ColumnName("version"),
DataType: sqlschema.DataTypeBigInt,
Nullable: false,
Default: "0",
}, 0)...)
mapperTable, mapperUniqueConstraints, err := migration.sqlschema.GetTable(ctx, sqlschema.TableName("span_mapper"))
if err != nil {
return err
}
sqls = append(sqls, migration.sqlschema.Operator().AddColumn(mapperTable, mapperUniqueConstraints, &sqlschema.Column{
Name: sqlschema.ColumnName("origin"),
DataType: sqlschema.DataTypeText,
Nullable: false,
Default: "'user'",
}, "user")...)
for _, sql := range sqls {
if _, err := tx.ExecContext(ctx, string(sql)); err != nil {
return err
}
}
if err := tx.Commit(); err != nil {
return err
}
return migration.sqlschema.ToggleFKEnforcement(ctx, db, true)
}
func (migration *addSpanMapperOrigin) Down(context.Context, *bun.DB) error {
return nil
}

View File

@@ -1,8 +1,6 @@
package spantypes
import (
"slices"
"strings"
"time"
"github.com/SigNoz/signoz/pkg/errors"
@@ -13,7 +11,6 @@ import (
var (
ErrCodeMapperNotFound = errors.MustNewCode("span_attribute_mapper_not_found")
ErrCodeMapperAlreadyExists = errors.MustNewCode("span_attribute_mapper_already_exists")
ErrCodeMapperNotDeletable = errors.MustNewCode("span_attribute_mapper_not_deletable")
ErrCodeMappingInvalidInput = errors.MustNewCode("span_attribute_mapping_invalid_input")
)
@@ -37,25 +34,12 @@ var (
SpanMapperOperationCopy = SpanMapperOperation{valuer.NewString("copy")}
)
// SpanMapperOrigin tells shipped (system) items apart from user-created ones.
// System items are read-only apart from their enabled toggle.
type SpanMapperOrigin struct {
valuer.String
}
var (
SpanMapperOriginUser = SpanMapperOrigin{valuer.NewString("user")}
SpanMapperOriginSystem = SpanMapperOrigin{valuer.NewString("system")}
)
// MapperSource describes one candidate source for a target attribute.
type SpanMapperSource struct {
Key string `json:"key" required:"true"`
Context FieldContext `json:"context" required:"true"`
Operation SpanMapperOperation `json:"operation" required:"true"`
Priority int `json:"priority" required:"true"`
Enabled bool `json:"enabled" required:"true"`
Origin SpanMapperOrigin `json:"origin"`
}
// MapperConfig holds the mapping logic for a single target attribute.
@@ -75,7 +59,6 @@ type SpanMapper struct {
FieldContext FieldContext `json:"fieldContext" required:"true"`
Config SpanMapperConfig `json:"config" required:"true"`
Enabled bool `json:"enabled" required:"true"`
Origin SpanMapperOrigin `json:"origin" required:"true"`
}
type PostableSpanMapper struct {
@@ -107,63 +90,6 @@ func (SpanMapperOperation) Enum() []any {
return []any{SpanMapperOperationMove, SpanMapperOperationCopy}
}
func (SpanMapperOrigin) Enum() []any {
return []any{SpanMapperOriginUser, SpanMapperOriginSystem}
}
func (p *PostableSpanMapper) Validate() error {
if strings.TrimSpace(p.Name) == "" {
return errors.New(errors.TypeInvalidInput, ErrCodeMappingInvalidInput, "mapper name must not be blank")
}
if err := p.FieldContext.Validate(); err != nil {
return err
}
return p.Config.Validate()
}
func (f FieldContext) Validate() error {
if f != FieldContextSpanAttribute && f != FieldContextResource {
return errors.Newf(errors.TypeInvalidInput, ErrCodeMappingInvalidInput, "field context must be one of %q or %q, got %q", FieldContextSpanAttribute, FieldContextResource, f.StringValue())
}
return nil
}
// Validate checks every source and rejects duplicate priorities within an
// origin. Shipped and user sources are never compared with each other: a user
// re-adding a shipped key with another operation is the supported override.
func (c *SpanMapperConfig) Validate() error {
if len(c.Sources) == 0 {
return errors.New(errors.TypeInvalidInput, ErrCodeMappingInvalidInput, "config.sources must contain at least one source")
}
seen := map[SpanMapperOrigin]map[int]struct{}{}
for _, s := range c.Sources {
if strings.TrimSpace(s.Key) == "" {
return errors.New(errors.TypeInvalidInput, ErrCodeMappingInvalidInput, "source key must not be blank")
}
if err := s.Context.Validate(); err != nil {
return err
}
if s.Operation != SpanMapperOperationCopy && s.Operation != SpanMapperOperationMove {
return errors.Newf(errors.TypeInvalidInput, ErrCodeMappingInvalidInput, "source operation must be one of %q or %q, got %q", SpanMapperOperationCopy, SpanMapperOperationMove, s.Operation.StringValue())
}
if !s.Origin.IsZero() && s.Origin != SpanMapperOriginUser && s.Origin != SpanMapperOriginSystem {
return errors.Newf(errors.TypeInvalidInput, ErrCodeMappingInvalidInput, "source origin must be one of %q or %q, got %q", SpanMapperOriginUser, SpanMapperOriginSystem, s.Origin.StringValue())
}
origin := s.Origin
if origin.IsZero() {
origin = SpanMapperOriginUser
}
if seen[origin] == nil {
seen[origin] = map[int]struct{}{}
}
if _, dup := seen[origin][s.Priority]; dup {
return errors.Newf(errors.TypeInvalidInput, ErrCodeMappingInvalidInput, "source priority %d is used more than once", s.Priority)
}
seen[origin][s.Priority] = struct{}{}
}
return nil
}
func NewSpanMapper(groupID valuer.UUID, createdBy string, p *PostableSpanMapper) *SpanMapper {
now := time.Now()
return &SpanMapper{
@@ -171,9 +97,8 @@ func NewSpanMapper(groupID valuer.UUID, createdBy string, p *PostableSpanMapper)
GroupID: groupID,
Name: p.Name,
FieldContext: p.FieldContext,
Config: SpanMapperConfig{Sources: withOrigin(p.Config.Sources, SpanMapperOriginUser)},
Config: p.Config,
Enabled: p.Enabled,
Origin: SpanMapperOriginUser,
TimeAuditable: types.TimeAuditable{
CreatedAt: now,
UpdatedAt: now,
@@ -185,42 +110,16 @@ func NewSpanMapper(groupID valuer.UUID, createdBy string, p *PostableSpanMapper)
}
}
// Update applies a user edit; a zero fieldContext means it was omitted. On a
// system mapper the field context is fixed and the stored system sources are
// kept; see nextSources.
func (m *SpanMapper) Update(fieldContext FieldContext, config *SpanMapperConfig, enabled *bool, updatedBy string) error {
if !fieldContext.IsZero() {
if m.Origin == SpanMapperOriginSystem && fieldContext != m.FieldContext {
return errors.Newf(errors.TypeInvalidInput, ErrCodeMappingInvalidInput, "field context of system mapper %q cannot be changed", m.Name)
}
if err := fieldContext.Validate(); err != nil {
return err
}
m.FieldContext = fieldContext
}
func (m *SpanMapper) Update(fieldContext FieldContext, config *SpanMapperConfig, enabled *bool, updatedBy string) {
m.FieldContext = fieldContext
if config != nil {
sources, err := m.nextSources(config.Sources)
if err != nil {
return err
}
m.Config = SpanMapperConfig{Sources: sources}
if err := m.Config.Validate(); err != nil {
return err
}
m.Config = *config
}
if enabled != nil {
m.Enabled = *enabled
}
m.UpdatedAt = time.Now()
m.UpdatedBy = updatedBy
return nil
}
func (m *SpanMapper) ErrIfNotDeletable() error {
if m.Origin == SpanMapperOriginSystem {
return errors.Newf(errors.TypeInvalidInput, ErrCodeMapperNotDeletable, "system mapper %q cannot be deleted, disable it instead", m.Name)
}
return nil
}
func (m *SpanMapper) ToStorable() *StorableSpanMapper {
@@ -233,7 +132,6 @@ func (m *SpanMapper) ToStorable() *StorableSpanMapper {
FieldContext: m.FieldContext,
Config: m.Config,
Enabled: m.Enabled,
Origin: m.Origin,
}
}
@@ -247,7 +145,6 @@ func (s *StorableSpanMapper) ToSpanMapper() *SpanMapper {
FieldContext: s.FieldContext,
Config: s.Config,
Enabled: s.Enabled,
Origin: s.Origin,
}
}
@@ -262,39 +159,3 @@ func NewSpanMappersFromStorable(ss []*StorableSpanMapper) []*SpanMapper {
func NewGettableSpanMappers(m []*SpanMapper) *GettableSpanMappers {
return &GettableSpanMappers{Items: m}
}
// nextSources builds the source list from an edit: user sources are taken from
// the edit as sent, system sources stay as stored and the edit can only flip
// their enabled flag.
func (m *SpanMapper) nextSources(edit []SpanMapperSource) ([]SpanMapperSource, error) {
var systemSources, userSources []SpanMapperSource
for _, s := range m.Config.Sources {
if s.Origin == SpanMapperOriginSystem {
systemSources = append(systemSources, s)
}
}
for _, s := range edit {
if s.Origin != SpanMapperOriginSystem {
s.Origin = SpanMapperOriginUser
userSources = append(userSources, s)
continue
}
idx := slices.IndexFunc(systemSources, func(o SpanMapperSource) bool { return o.Key == s.Key && o.Context == s.Context })
if idx == -1 {
return nil, errors.Newf(errors.TypeInvalidInput, ErrCodeMappingInvalidInput, "system source %q does not exist on this mapper; only its enabled flag can change", s.Key)
}
systemSources[idx].Enabled = s.Enabled
}
return append(systemSources, userSources...), nil
}
func withOrigin(sources []SpanMapperSource, origin SpanMapperOrigin) []SpanMapperSource {
out := make([]SpanMapperSource, len(sources))
for i, s := range sources {
s.Origin = origin
out[i] = s
}
return out
}

View File

@@ -1,8 +1,6 @@
package spantypes
import (
"slices"
"strings"
"time"
"github.com/SigNoz/signoz/pkg/errors"
@@ -13,28 +11,17 @@ import (
var (
ErrCodeMappingGroupNotFound = errors.MustNewCode("span_attribute_mapping_group_not_found")
ErrCodeMappingGroupAlreadyExists = errors.MustNewCode("span_attribute_mapping_group_already_exists")
ErrCodeMappingGroupNameReserved = errors.MustNewCode("span_attribute_mapping_group_name_reserved")
ErrCodeMappingGroupNotDeletable = errors.MustNewCode("span_attribute_mapping_group_not_deletable")
)
// SpanMapperGroupConditionKey is one substring a span's attribute or resource
// keys are matched against.
type SpanMapperGroupConditionKey struct {
Value string `json:"value" required:"true"`
Enabled bool `json:"enabled" required:"true"`
Origin SpanMapperOrigin `json:"origin"`
}
// SpanMapperGroupCondition gates whether a group's rules run for a given span.
// A group runs when any attribute or resource key on the span CONTAINS one of
// the listed substrings (plain substring match — no glob syntax).
type SpanMapperGroupCondition struct {
Attributes []SpanMapperGroupConditionKey `json:"attributes" required:"true" nullable:"true"`
Resource []SpanMapperGroupConditionKey `json:"resource" required:"true" nullable:"true"`
Attributes []string `json:"attributes" required:"true" nullable:"true"`
Resource []string `json:"resource" required:"true" nullable:"true"`
}
// SpanMapperGroup is the domain model for a span attribute mapping group.
// Version is the shipped definition version for system groups and 0 otherwise.
type SpanMapperGroup struct {
types.TimeAuditable
types.UserAuditable
@@ -44,8 +31,6 @@ type SpanMapperGroup struct {
Name string `json:"name" required:"true"`
Condition SpanMapperGroupCondition `json:"condition" required:"true"`
Enabled bool `json:"enabled" required:"true"`
Origin SpanMapperOrigin `json:"origin" required:"true"`
Version int `json:"version" required:"true"`
}
// GettableSpanMapperGroup is the HTTP response representation of a mapping group.
@@ -73,42 +58,14 @@ type GettableSpanMapperGroups struct {
Items []*GettableSpanMapperGroup `json:"items" required:"true" nullable:"false"`
}
// Validate requires at least one substring overall and rejects blank ones.
// All-off is allowed: a group with every substring disabled simply never runs.
func (c *SpanMapperGroupCondition) Validate() error {
if len(c.Attributes)+len(c.Resource) == 0 {
return errors.New(errors.TypeInvalidInput, ErrCodeMappingInvalidInput, "condition must list at least one attribute or resource substring")
}
for _, k := range slices.Concat(c.Attributes, c.Resource) {
if strings.TrimSpace(k.Value) == "" {
return errors.New(errors.TypeInvalidInput, ErrCodeMappingInvalidInput, "condition substrings must not be blank")
}
if !k.Origin.IsZero() && k.Origin != SpanMapperOriginUser && k.Origin != SpanMapperOriginSystem {
return errors.Newf(errors.TypeInvalidInput, ErrCodeMappingInvalidInput, "condition origin must be one of %q or %q, got %q", SpanMapperOriginUser, SpanMapperOriginSystem, k.Origin.StringValue())
}
}
return nil
}
func (p *PostableSpanMapperGroup) Validate() error {
if strings.TrimSpace(p.Name) == "" {
return errors.New(errors.TypeInvalidInput, ErrCodeMappingInvalidInput, "group name must not be blank")
}
return p.Condition.Validate()
}
func NewSpanMapperGroup(orgID valuer.UUID, createdBy string, p *PostableSpanMapperGroup) *SpanMapperGroup {
now := time.Now()
return &SpanMapperGroup{
ID: valuer.GenerateUUID(),
OrgID: orgID,
Name: p.Name,
Condition: SpanMapperGroupCondition{
Attributes: conditionKeysWithOrigin(p.Condition.Attributes, SpanMapperOriginUser),
Resource: conditionKeysWithOrigin(p.Condition.Resource, SpanMapperOriginUser),
},
Enabled: p.Enabled,
Origin: SpanMapperOriginUser,
ID: valuer.GenerateUUID(),
OrgID: orgID,
Name: p.Name,
Condition: p.Condition,
Enabled: p.Enabled,
TimeAuditable: types.TimeAuditable{
CreatedAt: now,
UpdatedAt: now,
@@ -120,45 +77,18 @@ func NewSpanMapperGroup(orgID valuer.UUID, createdBy string, p *PostableSpanMapp
}
}
// Update applies a user edit. A system group keeps its name and its system
// substrings; see nextConditionKeys.
func (g *SpanMapperGroup) Update(name *string, condition *SpanMapperGroupCondition, enabled *bool, updatedBy string) error {
func (g *SpanMapperGroup) Update(name *string, condition *SpanMapperGroupCondition, enabled *bool, updatedBy string) {
if name != nil {
if g.Origin == SpanMapperOriginSystem && *name != g.Name {
return errors.Newf(errors.TypeInvalidInput, ErrCodeMappingInvalidInput, "system group %q cannot be renamed", g.Name)
}
if strings.TrimSpace(*name) == "" {
return errors.New(errors.TypeInvalidInput, ErrCodeMappingInvalidInput, "group name must not be blank")
}
g.Name = *name
}
if condition != nil {
attrs, err := nextConditionKeys(g.Condition.Attributes, condition.Attributes)
if err != nil {
return err
}
res, err := nextConditionKeys(g.Condition.Resource, condition.Resource)
if err != nil {
return err
}
g.Condition = SpanMapperGroupCondition{Attributes: attrs, Resource: res}
if err := g.Condition.Validate(); err != nil {
return err
}
g.Condition = *condition
}
if enabled != nil {
g.Enabled = *enabled
}
g.UpdatedAt = time.Now()
g.UpdatedBy = updatedBy
return nil
}
func (g *SpanMapperGroup) ErrIfNotDeletable() error {
if g.Origin == SpanMapperOriginSystem {
return errors.Newf(errors.TypeInvalidInput, ErrCodeMappingGroupNotDeletable, "system group %q cannot be deleted, disable it instead", g.Name)
}
return nil
}
func (g *SpanMapperGroup) ToStorable() *StorableSpanMapperGroup {
@@ -170,8 +100,6 @@ func (g *SpanMapperGroup) ToStorable() *StorableSpanMapperGroup {
Name: g.Name,
Condition: g.Condition,
Enabled: g.Enabled,
Origin: g.Origin,
Version: g.Version,
}
}
@@ -184,8 +112,6 @@ func (s *StorableSpanMapperGroup) ToSpanMapperGroup() *SpanMapperGroup {
Name: s.Name,
Condition: s.Condition,
Enabled: s.Enabled,
Origin: s.Origin,
Version: s.Version,
}
}
@@ -200,39 +126,3 @@ func NewSpanMapperGroupsFromStorable(ss []*StorableSpanMapperGroup) []*SpanMappe
func NewGettableSpanMapperGroups(g []*SpanMapperGroup) *GettableSpanMapperGroups {
return &GettableSpanMapperGroups{Items: g}
}
// nextConditionKeys builds a substring list from an edit: user substrings are
// taken from the edit as sent, system substrings stay as stored and the edit
// can only flip their enabled flag.
func nextConditionKeys(stored, edit []SpanMapperGroupConditionKey) ([]SpanMapperGroupConditionKey, error) {
var systemKeys, userKeys []SpanMapperGroupConditionKey
for _, k := range stored {
if k.Origin == SpanMapperOriginSystem {
systemKeys = append(systemKeys, k)
}
}
for _, k := range edit {
if k.Origin != SpanMapperOriginSystem {
k.Origin = SpanMapperOriginUser
userKeys = append(userKeys, k)
continue
}
idx := slices.IndexFunc(systemKeys, func(s SpanMapperGroupConditionKey) bool { return s.Value == k.Value })
if idx == -1 {
return nil, errors.Newf(errors.TypeInvalidInput, ErrCodeMappingInvalidInput, "system substring %q does not exist on this group; only its enabled flag can change", k.Value)
}
systemKeys[idx].Enabled = k.Enabled
}
return append(systemKeys, userKeys...), nil
}
func conditionKeysWithOrigin(keys []SpanMapperGroupConditionKey, origin SpanMapperOrigin) []SpanMapperGroupConditionKey {
out := make([]SpanMapperGroupConditionKey, len(keys))
for i, k := range keys {
k.Origin = origin
out[i] = k
}
return out
}

View File

@@ -1,121 +0,0 @@
package spantypes
import (
"bytes"
"encoding/json"
"slices"
"strings"
"github.com/SigNoz/signoz/pkg/errors"
)
var ErrCodeMappingDefinitionInvalid = errors.MustNewCode("span_attribute_mapping_definition_invalid")
// ProvisionerIdentity is stamped into created_by/updated_by by the reconciler.
const ProvisionerIdentity = "signoz"
// SpanMapperGroupDefinition is one shipped mapping group. Version is bumped on
// every content change and drives upgrades; the group name is the stable key
// and never changes. Once parsed, every substring and source carries the
// system origin and is enabled.
type SpanMapperGroupDefinition struct {
Version int `json:"version"`
Definition PostableSpanMapperTestGroup `json:"definition"`
}
func (d SpanMapperGroupDefinition) Name() string {
return d.Definition.Name
}
func NewSpanMapperGroupDefinition(raw []byte) (SpanMapperGroupDefinition, error) {
decoder := json.NewDecoder(bytes.NewReader(raw))
decoder.DisallowUnknownFields()
var d SpanMapperGroupDefinition
if err := decoder.Decode(&d); err != nil {
return SpanMapperGroupDefinition{}, errors.WrapInvalidInputf(err, ErrCodeMappingDefinitionInvalid, "%s", err.Error())
}
if err := d.validate(); err != nil {
return SpanMapperGroupDefinition{}, err
}
for _, keys := range [][]SpanMapperGroupConditionKey{d.Definition.Condition.Attributes, d.Definition.Condition.Resource} {
for i := range keys {
keys[i].Enabled = true
keys[i].Origin = SpanMapperOriginSystem
}
}
for i := range d.Definition.Mappers {
sources := d.Definition.Mappers[i].Config.Sources
for j := range sources {
sources[j].Enabled = true
sources[j].Origin = SpanMapperOriginSystem
}
}
return d, nil
}
// SpanMapperGroupRegistry holds every definition embedded in the binary, keyed by name.
type SpanMapperGroupRegistry struct {
definitions map[string]SpanMapperGroupDefinition
}
func NewSpanMapperGroupRegistry(definitions []SpanMapperGroupDefinition) (SpanMapperGroupRegistry, error) {
byName := make(map[string]SpanMapperGroupDefinition, len(definitions))
for _, d := range definitions {
if _, dup := byName[d.Name()]; dup {
return SpanMapperGroupRegistry{}, errors.NewInvalidInputf(ErrCodeMappingDefinitionInvalid, "duplicate span mapper group name %q", d.Name())
}
byName[d.Name()] = d
}
return SpanMapperGroupRegistry{definitions: byName}, nil
}
func (r SpanMapperGroupRegistry) IsReserved(name string) bool {
_, ok := r.definitions[name]
return ok
}
// List returns the definitions sorted by name so provisioning order is stable.
func (r SpanMapperGroupRegistry) List() []SpanMapperGroupDefinition {
out := make([]SpanMapperGroupDefinition, 0, len(r.definitions))
for _, d := range r.definitions {
out = append(out, d)
}
slices.SortFunc(out, func(a, b SpanMapperGroupDefinition) int { return strings.Compare(a.Name(), b.Name()) })
return out
}
func (d SpanMapperGroupDefinition) validate() error {
if d.Version < 1 {
return errors.NewInvalidInputf(ErrCodeMappingDefinitionInvalid, "version must be at least 1, got %d", d.Version)
}
if err := d.Definition.Validate(); err != nil {
return errors.Wrapf(err, errors.TypeInvalidInput, ErrCodeMappingDefinitionInvalid, "%s", d.Name())
}
if len(d.Definition.Mappers) == 0 {
return errors.NewInvalidInputf(ErrCodeMappingDefinitionInvalid, "%s: at least one mapper is required", d.Name())
}
names := make(map[string]struct{}, len(d.Definition.Mappers))
for i := range d.Definition.Mappers {
m := &d.Definition.Mappers[i]
if err := m.Validate(); err != nil {
return errors.Wrapf(err, errors.TypeInvalidInput, ErrCodeMappingDefinitionInvalid, "%s: mapper %q", d.Name(), m.Name)
}
if _, dup := names[m.Name]; dup {
return errors.NewInvalidInputf(ErrCodeMappingDefinitionInvalid, "%s: duplicate mapper %q", d.Name(), m.Name)
}
names[m.Name] = struct{}{}
for _, s := range m.Config.Sources {
if !s.Origin.IsZero() || s.Enabled {
return errors.NewInvalidInputf(ErrCodeMappingDefinitionInvalid, "%s: mapper %q: sources must not set origin or enabled", d.Name(), m.Name)
}
}
}
for _, k := range slices.Concat(d.Definition.Condition.Attributes, d.Definition.Condition.Resource) {
if !k.Origin.IsZero() || k.Enabled {
return errors.NewInvalidInputf(ErrCodeMappingDefinitionInvalid, "%s: condition substrings must not set origin or enabled", d.Name())
}
}
return nil
}

View File

@@ -86,31 +86,17 @@ func buildProcessorConfig(groups []*SpanMapperGroupWithMappers) *spanMapperProce
out := make([]spanMapperProcessorGroup, 0, len(groups))
for _, gm := range groups {
existsAny := spanMapperProcessorExistsAny{
Attributes: enabledConditionValues(gm.Group.Condition.Attributes),
Resource: enabledConditionValues(gm.Group.Condition.Resource),
}
// The collector rejects an empty exists_any and empty sources; with
// per-item toggles, all-off is valid stored state and means "never runs".
if len(existsAny.Attributes)+len(existsAny.Resource) == 0 {
continue
}
rules := make([]spanMapperProcessorAttribute, 0, len(gm.Mappers))
for _, m := range gm.Mappers {
rule := buildAttributeRule(m)
if len(rule.Sources) == 0 {
continue
}
rules = append(rules, rule)
}
if len(rules) == 0 {
continue
rules = append(rules, buildAttributeRule(m))
}
out = append(out, spanMapperProcessorGroup{
ID: gm.Group.Name,
ExistsAny: existsAny,
ID: gm.Group.Name,
ExistsAny: spanMapperProcessorExistsAny{
Attributes: gm.Group.Condition.Attributes,
Resource: gm.Group.Condition.Resource,
},
Attributes: rules,
})
}
@@ -118,31 +104,14 @@ func buildProcessorConfig(groups []*SpanMapperGroupWithMappers) *spanMapperProce
return &spanMapperProcessorConfig{Groups: out}
}
func enabledConditionValues(keys []SpanMapperGroupConditionKey) []string {
out := make([]string, 0, len(keys))
for _, k := range keys {
if k.Enabled {
out = append(out, k.Value)
}
}
if len(out) == 0 {
return nil
}
return out
}
// buildAttributeRule maps a single SpanMapper to a collector attribute rule.
// Disabled sources are skipped and the rest are sorted by Priority DESC
// (highest-priority first); read-from-resource sources are encoded via the
// "resource." prefix on the key. Each source carries its own action — "copy"
// is omitted to keep the emitted YAML compact, and only "move" is set explicitly.
// Sources are sorted by Priority DESC (highest-priority first); read-from-
// resource sources are encoded via the "resource." prefix on the key. Each
// source carries its own action — "copy" is omitted to keep the emitted YAML
// compact, and only "move" is set explicitly.
func buildAttributeRule(m *SpanMapper) spanMapperProcessorAttribute {
sources := make([]SpanMapperSource, 0, len(m.Config.Sources))
for _, s := range m.Config.Sources {
if s.Enabled {
sources = append(sources, s)
}
}
sources := make([]SpanMapperSource, len(m.Config.Sources))
copy(sources, m.Config.Sources)
sort.SliceStable(sources, func(i, j int) bool { return sources[i].Priority > sources[j].Priority })
out := make([]spanMapperProcessorSource, 0, len(sources))

View File

@@ -145,22 +145,6 @@ func TestBuildAttributeRule(t *testing.T) {
},
},
},
{
name: "disabled_sources_skipped",
mapper: newMapper("gen_ai.input.messages", FieldContextSpanAttribute,
systemSrc("gen_ai.prompt", SpanMapperOperationCopy, 30, false),
systemSrc("input.value", SpanMapperOperationCopy, 20, true),
attrSrc("gen_ai.prompt", SpanMapperOperationMove, 40),
),
want: spanMapperProcessorAttribute{
Target: "gen_ai.input.messages",
Context: FieldContextSpanAttribute.StringValue(),
Sources: []spanMapperProcessorSource{
{Key: "gen_ai.prompt", Action: SpanMapperOperationMove.StringValue()},
{Key: "input.value"},
},
},
},
}
for _, tc := range tests {
@@ -171,33 +155,6 @@ func TestBuildAttributeRule(t *testing.T) {
}
}
func TestBuildProcessorConfigDropsAllOffItems(t *testing.T) {
t.Parallel()
offGroup := newGroup("all-off", nil, nil)
offGroup.Condition.Attributes = []SpanMapperGroupConditionKey{{Value: "model", Enabled: false, Origin: SpanMapperOriginSystem}}
mixed := newGroup("llm", nil, nil)
mixed.Condition.Attributes = []SpanMapperGroupConditionKey{
{Value: "model", Enabled: false, Origin: SpanMapperOriginSystem},
{Value: "gen_ai.request.model", Enabled: true, Origin: SpanMapperOriginUser},
}
got := buildProcessorConfig([]*SpanMapperGroupWithMappers{
{Group: offGroup, Mappers: []*SpanMapper{newMapper("gen_ai.request.model", FieldContextSpanAttribute, attrSrc("llm.model", SpanMapperOperationCopy, 1))}},
{Group: mixed, Mappers: []*SpanMapper{
newMapper("gen_ai.request.model", FieldContextSpanAttribute, systemSrc("llm.model", SpanMapperOperationCopy, 10, false)),
newMapper("gen_ai.provider.name", FieldContextSpanAttribute, systemSrc("llm.vendor", SpanMapperOperationCopy, 10, true)),
}},
})
require.Len(t, got.Groups, 1)
assert.Equal(t, "llm", got.Groups[0].ID)
assert.Equal(t, []string{"gen_ai.request.model"}, got.Groups[0].ExistsAny.Attributes)
require.Len(t, got.Groups[0].Attributes, 1)
assert.Equal(t, "gen_ai.provider.name", got.Groups[0].Attributes[0].Target)
}
func loadFixture(t *testing.T, name string) []byte {
t.Helper()
b, err := os.ReadFile(filepath.Join("testdata", name))
@@ -217,23 +174,12 @@ func assertYAMLEqual(t *testing.T, want, got []byte) {
func newGroup(name string, attrs, res []string) *SpanMapperGroup {
return &SpanMapperGroup{
Name: name,
Condition: SpanMapperGroupCondition{
Attributes: userConditionKeys(attrs),
Resource: userConditionKeys(res),
},
Enabled: true,
Name: name,
Condition: SpanMapperGroupCondition{Attributes: attrs, Resource: res},
Enabled: true,
}
}
func userConditionKeys(values []string) []SpanMapperGroupConditionKey {
out := make([]SpanMapperGroupConditionKey, len(values))
for i, v := range values {
out[i] = SpanMapperGroupConditionKey{Value: v, Enabled: true, Origin: SpanMapperOriginUser}
}
return out
}
func newMapper(name string, target FieldContext, sources ...SpanMapperSource) *SpanMapper {
return &SpanMapper{
Name: name,
@@ -244,13 +190,9 @@ func newMapper(name string, target FieldContext, sources ...SpanMapperSource) *S
}
func attrSrc(key string, op SpanMapperOperation, priority int) SpanMapperSource {
return SpanMapperSource{Key: key, Context: FieldContextSpanAttribute, Operation: op, Priority: priority, Enabled: true, Origin: SpanMapperOriginUser}
return SpanMapperSource{Key: key, Context: FieldContextSpanAttribute, Operation: op, Priority: priority}
}
func resSrc(key string, op SpanMapperOperation, priority int) SpanMapperSource {
return SpanMapperSource{Key: key, Context: FieldContextResource, Operation: op, Priority: priority, Enabled: true, Origin: SpanMapperOriginUser}
}
func systemSrc(key string, op SpanMapperOperation, priority int, enabled bool) SpanMapperSource {
return SpanMapperSource{Key: key, Context: FieldContextSpanAttribute, Operation: op, Priority: priority, Enabled: enabled, Origin: SpanMapperOriginSystem}
return SpanMapperSource{Key: key, Context: FieldContextResource, Operation: op, Priority: priority}
}

View File

@@ -15,14 +15,14 @@ func TestSimulateSpanMappersProcessing_EndToEnd(t *testing.T) {
groups := []*SpanMapperGroupWithMappers{{
Group: &SpanMapperGroup{
Name: "llm",
Condition: SpanMapperGroupCondition{Attributes: userConditionKeys([]string{"model"})},
Condition: SpanMapperGroupCondition{Attributes: []string{"model"}},
Enabled: true,
},
Mappers: []*SpanMapper{{
Name: "gen_ai.request.model",
FieldContext: FieldContextSpanAttribute,
Config: SpanMapperConfig{Sources: []SpanMapperSource{
{Key: "llm.model", Context: FieldContextSpanAttribute, Operation: SpanMapperOperationCopy, Priority: 1, Enabled: true, Origin: SpanMapperOriginUser},
{Key: "llm.model", Context: FieldContextSpanAttribute, Operation: SpanMapperOperationCopy, Priority: 1},
}},
Enabled: true,
}},

View File

@@ -20,9 +20,7 @@ type StorableSpanMapperGroup struct {
OrgID valuer.UUID `bun:"org_id,type:text,notnull"`
Name string `bun:"name,type:text,notnull"`
Condition SpanMapperGroupCondition `bun:"condition,type:jsonb,notnull"`
Enabled bool `bun:"enabled,notnull"`
Origin SpanMapperOrigin `bun:"origin,type:text,notnull"`
Version int `bun:"version,notnull"`
Enabled bool `bun:"enabled,notnull,default:true"`
}
type StorableSpanMapper struct {
@@ -36,8 +34,7 @@ type StorableSpanMapper struct {
Name string `bun:"name,type:text,notnull"`
FieldContext FieldContext `bun:"field_context,type:text,notnull"`
Config SpanMapperConfig `bun:"config,type:jsonb,notnull"`
Enabled bool `bun:"enabled,notnull"`
Origin SpanMapperOrigin `bun:"origin,type:text,notnull"`
Enabled bool `bun:"enabled,notnull,default:true"`
}
func (c SpanMapperGroupCondition) Value() (driver.Value, error) {

View File

@@ -9,14 +9,9 @@ import (
)
type SpanMapperStore interface {
// RunInTx runs cb in one transaction; every store call made with the
// callback's ctx joins it.
RunInTx(ctx context.Context, cb func(ctx context.Context) error) error
// Group operations
ListGroups(ctx context.Context, orgID valuer.UUID, q *ListSpanMapperGroupsQuery) ([]*SpanMapperGroup, error)
GetGroup(ctx context.Context, orgID, id valuer.UUID) (*SpanMapperGroup, error)
GetGroupByName(ctx context.Context, orgID valuer.UUID, name string) (*SpanMapperGroup, error)
CreateGroup(ctx context.Context, group *SpanMapperGroup) error
UpdateGroup(ctx context.Context, group *SpanMapperGroup) error
DeleteGroup(ctx context.Context, orgID, id valuer.UUID) error

View File

@@ -10,6 +10,7 @@ from fixtures import types
from fixtures.auth import (
USER_ADMIN_EMAIL,
USER_ADMIN_PASSWORD,
create_active_user,
)
TIMEOUT = 10

View File

@@ -41,7 +41,7 @@ def test_create_groups_and_simulate_with_backfill(
},
json={
"name": "llm-backfill",
"condition": {"attributes": [{"value": "model", "enabled": True}], "resource": []},
"condition": {"attributes": ["model"], "resource": []},
"enabled": True,
},
)
@@ -69,7 +69,6 @@ def test_create_groups_and_simulate_with_backfill(
"context": "attribute",
"operation": "copy",
"priority": 1,
"enabled": True,
}
]
},
@@ -127,13 +126,13 @@ def test_create_groups_and_simulate_with_backfill(
# No "mappers" key: the server backfills them from the saved group.
{
"name": "llm-backfill",
"condition": {"attributes": [{"value": "model", "enabled": True}], "resource": []},
"condition": {"attributes": ["model"], "resource": []},
"enabled": True,
},
# Unsaved group; mappers provided inline.
{
"name": "db-inline",
"condition": {"attributes": [{"value": "db", "enabled": True}], "resource": []},
"condition": {"attributes": ["db"], "resource": []},
"enabled": True,
"mappers": [
{
@@ -146,7 +145,6 @@ def test_create_groups_and_simulate_with_backfill(
"context": "attribute",
"operation": "move",
"priority": 1,
"enabled": True,
}
]
},

View File

@@ -1,152 +0,0 @@
from collections.abc import Callable
from http import HTTPStatus
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
GROUPS_PATH = "/api/v1/span_mapper_groups"
def test_default_groups_are_seeded_and_shipped_items_are_toggle_only(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Setup:
A fresh org. The reconciler seeds the shipped mapping groups at startup
and on org creation, so nothing has to be created here.
Tests:
1. The list contains llm, agent and tool as system groups with shipped
substrings
2. Shipped mappers and their sources are system-owned and enabled
3. A shipped name cannot be taken by a user group, and system groups and
mappers cannot be deleted
4. A shipped source can be switched off and a user override added; both
round-trip through PATCH and the simulator honours them
5. A shipped substring can be switched off and a user one added
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
headers = {"authorization": f"Bearer {token}", "content-type": "application/json"}
list_groups = requests.get(signoz.self.host_configs["8080"].get(GROUPS_PATH), timeout=10, headers=headers)
assert list_groups.status_code == HTTPStatus.OK
groups = {g["name"]: g for g in list_groups.json()["data"]["items"]}
assert {"llm", "agent", "tool"} <= set(groups)
for name in ("llm", "agent", "tool"):
assert groups[name]["origin"] == "system"
assert groups[name]["version"] >= 1
assert groups[name]["createdBy"] == "signoz"
llm = groups["llm"]
assert llm["condition"]["attributes"] == [{"value": "model", "enabled": True, "origin": "system"}]
list_mappers = requests.get(signoz.self.host_configs["8080"].get(f"{GROUPS_PATH}/{llm['id']}/span_mappers"), timeout=10, headers=headers)
assert list_mappers.status_code == HTTPStatus.OK
mappers = {m["name"]: m for m in list_mappers.json()["data"]["items"]}
model = mappers["gen_ai.request.model"]
assert model["origin"] == "system"
assert model["enabled"] is True
assert all(s["origin"] == "system" and s["enabled"] is True for s in model["config"]["sources"])
assert "llm.model_name" in [s["key"] for s in model["config"]["sources"]]
reserved = requests.post(
signoz.self.host_configs["8080"].get(GROUPS_PATH),
timeout=10,
headers=headers,
json={"name": "tool", "condition": {"attributes": [{"value": "tool", "enabled": True}], "resource": []}, "enabled": True},
)
assert reserved.status_code == HTTPStatus.BAD_REQUEST
assert reserved.json()["error"]["code"] == "span_attribute_mapping_group_name_reserved"
delete_group = requests.delete(signoz.self.host_configs["8080"].get(f"{GROUPS_PATH}/{llm['id']}"), timeout=10, headers=headers)
assert delete_group.status_code == HTTPStatus.BAD_REQUEST
assert delete_group.json()["error"]["code"] == "span_attribute_mapping_group_not_deletable"
delete_mapper = requests.delete(signoz.self.host_configs["8080"].get(f"{GROUPS_PATH}/{llm['id']}/span_mappers/{model['id']}"), timeout=10, headers=headers)
assert delete_mapper.status_code == HTTPStatus.BAD_REQUEST
assert delete_mapper.json()["error"]["code"] == "span_attribute_mapper_not_deletable"
# Switch the shipped llm.model_name source off and re-add it as a user move.
sources = [{**s, "enabled": s["key"] != "llm.model_name"} for s in model["config"]["sources"]] + [{"key": "llm.model_name", "context": "attribute", "operation": "move", "priority": 1, "enabled": True}]
patch_mapper = requests.patch(
signoz.self.host_configs["8080"].get(f"{GROUPS_PATH}/{llm['id']}/span_mappers/{model['id']}"),
timeout=10,
headers=headers,
json={"config": {"sources": sources}},
)
assert patch_mapper.status_code == HTTPStatus.NO_CONTENT
list_mappers = requests.get(signoz.self.host_configs["8080"].get(f"{GROUPS_PATH}/{llm['id']}/span_mappers"), timeout=10, headers=headers)
updated = {m["name"]: m for m in list_mappers.json()["data"]["items"]}["gen_ai.request.model"]
by_origin = {(s["key"], s["origin"]): s for s in updated["config"]["sources"]}
assert by_origin[("llm.model_name", "system")]["enabled"] is False
assert by_origin[("llm.model_name", "user")]["operation"] == "move"
assert len(updated["config"]["sources"]) == len(model["config"]["sources"]) + 1
# The user override wins: the source is moved, not copied.
simulate = requests.post(
signoz.self.host_configs["8080"].get(f"{GROUPS_PATH}/test"),
timeout=10,
headers=headers,
json={
"spans": [{"attributes": {"llm.model_name": "gpt-4o"}, "resource": {}}],
"groups": [{"name": "llm", "condition": llm["condition"], "enabled": True}],
},
)
assert simulate.status_code == HTTPStatus.OK
attrs = simulate.json()["data"]["spans"][0]["attributes"]
assert attrs["gen_ai.request.model"] == "gpt-4o"
assert "llm.model_name" not in attrs
# A shipped substring that does not exist is rejected; toggling one is not.
bad_condition = requests.patch(
signoz.self.host_configs["8080"].get(f"{GROUPS_PATH}/{llm['id']}"),
timeout=10,
headers=headers,
json={"condition": {"attributes": [{"value": "nope", "enabled": True, "origin": "system"}], "resource": []}},
)
assert bad_condition.status_code == HTTPStatus.BAD_REQUEST
patch_group = requests.patch(
signoz.self.host_configs["8080"].get(f"{GROUPS_PATH}/{llm['id']}"),
timeout=10,
headers=headers,
json={
"condition": {
"attributes": [
{"value": "model", "enabled": False, "origin": "system"},
{"value": "gen_ai.request.model", "enabled": True},
],
"resource": [],
}
},
)
assert patch_group.status_code == HTTPStatus.NO_CONTENT
list_groups = requests.get(signoz.self.host_configs["8080"].get(GROUPS_PATH), timeout=10, headers=headers)
llm_after = {g["name"]: g for g in list_groups.json()["data"]["items"]}["llm"]
assert llm_after["condition"]["attributes"] == [
{"value": "model", "enabled": False, "origin": "system"},
{"value": "gen_ai.request.model", "enabled": True, "origin": "user"},
]
assert llm_after["origin"] == "system"
assert llm_after["name"] == "llm"
# Leave the shipped group as seeded for the other suites.
restore_group = requests.patch(
signoz.self.host_configs["8080"].get(f"{GROUPS_PATH}/{llm['id']}"),
timeout=10,
headers=headers,
json={"condition": {"attributes": [{"value": "model", "enabled": True, "origin": "system"}], "resource": []}},
)
assert restore_group.status_code == HTTPStatus.NO_CONTENT
restore_mapper = requests.patch(
signoz.self.host_configs["8080"].get(f"{GROUPS_PATH}/{llm['id']}/span_mappers/{model['id']}"),
timeout=10,
headers=headers,
json={"config": {"sources": model["config"]["sources"]}},
)
assert restore_mapper.status_code == HTTPStatus.NO_CONTENT