Compare commits

..

3 Commits

Author SHA1 Message Date
Tushar Vats
ff2e5e7791 fix: has function family revamp 2026-07-14 03:06:01 +05:30
Vinicius Lourenço
9476d8f70f feat(infrastructure-monitoring-v2): add button to open chart at metrics explorer (#12107)
Some checks failed
build-staging / prepare (push) Has been cancelled
build-staging / js-build (push) Has been cancelled
build-staging / go-build (push) Has been cancelled
build-staging / staging (push) Has been cancelled
Release Drafter / update_release_draft (push) Has been cancelled
2026-07-13 19:29:46 +00:00
Vinicius Lourenço
69a240fdb2 chore(frontend): bump deps (#12052)
* chore(frontend): bump deps

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* chore(pnpm-lock): missing update

---------

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
2026-07-13 17:14:22 +00:00
38 changed files with 3820 additions and 3098 deletions

View File

@@ -24405,17 +24405,9 @@ paths:
description: Internal Server Error
security:
- api_key:
- logs:read
- traces:read
- metrics:read
- audit-logs:read
- meter-metrics:read
- VIEWER
- tokenizer:
- logs:read
- traces:read
- metrics:read
- audit-logs:read
- meter-metrics:read
- VIEWER
summary: Query range
tags:
- querier
@@ -24482,17 +24474,9 @@ paths:
description: Internal Server Error
security:
- api_key:
- logs:read
- traces:read
- metrics:read
- audit-logs:read
- meter-metrics:read
- VIEWER
- tokenizer:
- logs:read
- traces:read
- metrics:read
- audit-logs:read
- meter-metrics:read
- VIEWER
summary: Query range preview
tags:
- querier

View File

@@ -223,12 +223,17 @@ func (provider *provider) Update(ctx context.Context, orgID valuer.UUID, updated
return err
}
desiredTuples, err := authtypes.NewTuplesFromTransactionGroups(existingRole.Name, orgID, updatedRole.TransactionGroups)
existingGroups := authtypes.MustNewTransactionGroupsFromTuples(existingTuples)
additions, deletions := existingGroups.Diff(updatedRole.TransactionGroups)
additionTuples, err := authtypes.NewTuplesFromTransactionGroups(existingRole.Name, orgID, additions)
if err != nil {
return err
}
additionTuples, deletionTuples := authtypes.DiffTuples(existingTuples, desiredTuples)
deletionTuples, err := authtypes.NewTuplesFromTransactionGroups(existingRole.Name, orgID, deletions)
if err != nil {
return err
}
err = provider.Write(ctx, additionTuples, deletionTuples)
if err != nil {

View File

@@ -1,29 +0,0 @@
import { PropsWithChildren } from 'react';
type CommonProps = PropsWithChildren<{
className?: string;
minSize?: number;
maxSize?: number;
defaultSize?: number;
direction?: 'horizontal' | 'vertical';
autoSaveId?: string;
withHandle?: boolean;
}>;
export function ResizablePanelGroup({
children,
className,
}: CommonProps): JSX.Element {
return <div className={className}>{children}</div>;
}
export function ResizablePanel({
children,
className,
}: CommonProps): JSX.Element {
return <div className={className}>{children}</div>;
}
export function ResizableHandle({ className }: CommonProps): JSX.Element {
return <div className={className} />;
}

View File

@@ -21,7 +21,6 @@ const config: Config.InitialOptions = {
'\\.md$': '<rootDir>/__mocks__/cssMock.ts',
'^uplot$': '<rootDir>/__mocks__/uplotMock.ts',
'^motion/react$': '<rootDir>/__mocks__/motionMock.tsx',
'^@signozhq/resizable$': '<rootDir>/__mocks__/resizableMock.tsx',
'^hooks/useSafeNavigate$': USE_SAFE_NAVIGATE_MOCK_PATH,
'^src/hooks/useSafeNavigate$': USE_SAFE_NAVIGATE_MOCK_PATH,
'^.*/useSafeNavigate$': USE_SAFE_NAVIGATE_MOCK_PATH,

View File

@@ -49,7 +49,6 @@
"@sentry/vite-plugin": "5.3.0",
"@signozhq/design-tokens": "2.1.4",
"@signozhq/icons": "0.4.0",
"@signozhq/resizable": "0.0.2",
"@signozhq/ui": "0.0.23",
"@tanstack/react-table": "8.21.3",
"@tanstack/react-virtual": "3.13.22",
@@ -128,7 +127,7 @@
"timestamp-nano": "^1.0.0",
"typescript": "5.9.3",
"uplot": "1.6.31",
"uuid": "^8.3.2",
"uuid": "14.0.1",
"vite": "npm:rolldown-vite@7.3.1",
"vite-plugin-html": "3.2.2",
"zod": "4.3.6",
@@ -222,34 +221,5 @@
"*.(scss|css)": [
"stylelint"
]
},
"resolutions": {
"@types/react": "18.0.26",
"@types/react-dom": "18.0.10",
"debug": "4.3.4",
"semver": "7.5.4",
"xml2js": "0.5.0",
"phin": "^3.7.1",
"body-parser": "1.20.3",
"http-proxy-middleware": "4.1.1",
"cross-spawn": "7.0.5",
"cookie": "^0.7.1",
"serialize-javascript": "6.0.2",
"prismjs": "1.30.0",
"got": "11.8.5",
"form-data": "4.0.6",
"brace-expansion": "^2.0.3",
"on-headers": "^1.1.0",
"js-cookie": "^3.0.7",
"tmp": "0.2.7",
"vite": "npm:rolldown-vite@7.3.1",
"dompurify": "3.4.11",
"js-yaml@3": "3.15.0",
"js-yaml@4": "4.2.0",
"yaml@1": "1.10.3",
"react-router@6": "6.30.4",
"markdown-it": "14.2.0",
"mdast-util-to-hast@13": "13.2.1",
"protocol-buffers-schema": "3.6.1"
}
}

1864
frontend/pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,53 @@
trustPolicy: no-downgrade
blockExoticSubdeps: true
minimumReleaseAge: 2880 # 2d
minimumReleaseAgeStrict: true
minimumReleaseAgeExclude:
- '@signozhq/*'
blockExoticSubdeps: true
minimumReleaseAgeStrict: true
# Security floors for vulnerable transitive deps. Where possible, targets are
# capped to avoid crossing breaking versions (major; and minor for 0.x).
# Some entries may still force a breaking bump when no safe release exists
# within the consumers constraint—see per-entry notes.
overrides:
# via: direct devDep @babel/core ^7.22.11 (+ babel plugin peers)
# remove: bump @babel/core in package.json to ^7.29.6
'@babel/core@<=7.29.0': '>=7.29.6 <8'
# via: jest > babel-plugin-istanbul > @istanbuljs/load-nyc-config@1.1.0 (js-yaml ^3.13.1)
# remove: blocked — 1.1.0 is latest and still depends on js-yaml 3.x
'@istanbuljs/load-nyc-config>js-yaml': '>=4.2.0 <5'
# via: msw@1.3.2 (devDep) > cookie ^0.4.2
# remove: upgrade msw to >=2 (ships cookie ^1). Do NOT open the cap: cookie >=1 is
# ESM-only and breaks msw under jest's CJS sandbox (kills every test suite)
cookie@<0.7.0: '>=0.7.1 <1'
# via: direct dep dompurify 3.4.0; @grafana/data@11.6.15 (3.4.0/3.2.4 exact);
# @monaco-editor/react > monaco-editor@0.55.1 (3.2.7 exact)
# remove: bump direct dep to 3.4.11; @grafana/data (latest 13.1.0) and
# monaco-editor (latest 0.55.1) still pin vulnerable versions — blocked
dompurify@<=3.4.10: '>=3.4.11 <4'
# via: rolldown-vite@7.3.1 (esbuild ^0.27.0); orval@8.9.1 (^0.27.4); ts-jest@29.4.9 (~0.27.4)
# remove: blocked on rolldown-vite (7.3.1 is latest, still ^0.27.0);
# orval >=8.20.0 and ts-jest >=29.4.11 already fixed on their side
esbuild@>=0.27.3 <0.28.1: '>=0.28.1 <0.29.0'
# via: react-use@17.5.1 (direct, js-cookie ^2.2.1); @grafana/data > react-use@17.6.0
# remove: bump react-use to >=17.6.1 (js-cookie ^3); @grafana/data side blocked
js-cookie@<=3.0.5: '>=3.0.7 <4'
# via: @orval/core@8.9.1 (devDep, js-yaml 4.1.1 EXACT pin — not deletable);
# json-schema-to-typescript@15 > @apidevtools/json-schema-ref-parser (^4.1.0)
# remove: upgrade orval to >=8.20.0 (drops js-yaml dependency entirely)
js-yaml@>=4.0.0 <=4.1.1: '>=4.2.0 <5'
# via: react-syntax-highlighter@15.5.0 (prismjs ^1.27.0 + refractor@3 ~1.27.0 tilde-pinned)
# remove: bump react-syntax-highlighter to >=16.1.1 (prismjs ^1.30.0, refractor@5)
prismjs@<1.30.0: '>=1.30.0 <2'
# via: direct dep react-router-dom-v5-compat@6.30.3 (react-router 6.30.3 exact)
# remove: bump react-router-dom-v5-compat to 6.30.4. Do NOT open the cap:
# react-router >=7 requires React 19 and breaks the app-wide CompatRouter
react-router@>=6.7.0 <6.30.4: '>=6.30.4 <7'
# via: msw@1.3.2 (devDep) > inquirer@8 > external-editor@3.1.0 (tmp ^0.0.33)
# remove: upgrade msw to >=2 (drops the inquirer/external-editor chain)
tmp@<0.2.6: '>=0.2.6 <0.3.0'
# via: jest > babel-plugin-macros > cosmiconfig@7 (yaml ^1.10.0);
# typescript-plugin-css-modules > postcss-load-config@3 (^1.10.2)
# remove: blocked — babel-plugin-macros 3.1.0 (latest) still uses cosmiconfig@7
yaml@>=1.0.0 <1.10.3: '>=1.10.3 <2'
trustPolicy: no-downgrade
trustPolicyExclude:
- 'semver@6.3.1 || 5.7.2'

View File

@@ -12,5 +12,4 @@
import '@signozhq/design-tokens';
import '@signozhq/icons';
import '@signozhq/resizable';
import '@signozhq/ui';

View File

@@ -14,11 +14,28 @@
box-sizing: border-box;
}
.entityMetricsTitleContainer {
display: flex;
align-items: center;
gap: 8px;
}
.entityMetricsTitle {
font-size: var(--periscope-font-size-base);
color: var(--l2-foreground);
}
.metricsExplorerLink {
display: flex;
align-items: center;
color: var(--l2-foreground);
transition: opacity 0.2s;
&:hover {
color: var(--l3-foreground);
}
}
.metricsHeader {
display: flex;
justify-content: flex-end;

View File

@@ -1,6 +1,8 @@
import { useCallback, useMemo, useRef } from 'react';
import { UseQueryResult } from 'react-query';
import { Skeleton } from 'antd';
import { Link } from 'react-router-dom';
import { Compass } from '@signozhq/icons';
import { Skeleton, Tooltip } from 'antd';
import cx from 'classnames';
import { PANEL_TYPES } from 'constants/queryBuilder';
import TimeSeries from 'container/DashboardContainer/visualization/charts/TimeSeries/TimeSeries';
@@ -19,6 +21,7 @@ import { GetQueryResultsProps } from 'lib/dashboard/getQueryResults';
import { useTimezone } from 'providers/Timezone';
import { SuccessResponse } from 'types/api';
import { MetricRangePayloadProps } from 'types/api/metrics/getQueryRange';
import { getMetricsExplorerUrl } from 'utils/explorerUtils';
import { buildEntityMetricsChartConfig } from './configBuilder';
@@ -204,9 +207,31 @@ function EntityMetrics<T>({
key={entityWidgetInfo[idx].title}
className={styles.entityMetricsCol}
>
<span className={styles.entityMetricsTitle}>
{entityWidgetInfo[idx].title}
</span>
<div className={styles.entityMetricsTitleContainer}>
<span className={styles.entityMetricsTitle}>
{entityWidgetInfo[idx].title}
</span>
{queryPayloads[idx] &&
queryPayloads[idx].graphType !== PANEL_TYPES.TABLE && (
<Tooltip title="Open in Metrics Explorer">
<Link
to={getMetricsExplorerUrl({
query: queryPayloads[idx].query,
...(selectedInterval && selectedInterval !== 'custom'
? { relativeTime: selectedInterval }
: {
startTimeMs: timeRange.startTime * 1000,
endTimeMs: timeRange.endTime * 1000,
}),
})}
className={styles.metricsExplorerLink}
data-testid={`open-metrics-explorer-${idx}`}
>
<Compass size={14} />
</Link>
</Tooltip>
)}
</div>
<div className={styles.entityMetricsCard} ref={graphRef}>
{renderCardContent(query, idx)}
</div>

View File

@@ -1,4 +1,5 @@
import { render, screen } from '@testing-library/react';
import { MemoryRouter } from 'react-router-dom';
import { InfraMonitoringEntity } from 'container/InfraMonitoringK8sV2/constants';
import { Time } from 'container/TopNav/DateTimeSelectionV2/types';
import * as appContextHooks from 'providers/App/App';
@@ -295,17 +296,19 @@ const renderEntityMetrics = (overrides = {}): any => {
};
return render(
<EntityMetrics
timeRange={defaultProps.timeRange}
isModalTimeSelection={defaultProps.isModalTimeSelection}
handleTimeChange={defaultProps.handleTimeChange}
selectedInterval={defaultProps.selectedInterval}
entity={defaultProps.entity}
entityWidgetInfo={defaultProps.entityWidgetInfo}
getEntityQueryPayload={defaultProps.getEntityQueryPayload}
queryKey={defaultProps.queryKey}
category={defaultProps.category}
/>,
<MemoryRouter>
<EntityMetrics
timeRange={defaultProps.timeRange}
isModalTimeSelection={defaultProps.isModalTimeSelection}
handleTimeChange={defaultProps.handleTimeChange}
selectedInterval={defaultProps.selectedInterval}
entity={defaultProps.entity}
entityWidgetInfo={defaultProps.entityWidgetInfo}
getEntityQueryPayload={defaultProps.getEntityQueryPayload}
queryKey={defaultProps.queryKey}
category={defaultProps.category}
/>
</MemoryRouter>,
);
};
@@ -334,8 +337,8 @@ const mockTableData: (import('../utils').MetricsTableData[] | null)[] = [
];
const mockQueryPayloads = [
{ graphType: 'graph' }, // time_series
{ graphType: 'table' }, // table
{ graphType: 'graph', query: { queryType: 'builder' } }, // time_series
{ graphType: 'table', query: { queryType: 'builder' } }, // table
];
describe('EntityMetrics', () => {
@@ -442,6 +445,34 @@ describe('EntityMetrics', () => {
);
});
it('renders metrics explorer link only for non-table panels', () => {
renderEntityMetrics();
expect(screen.getByTestId('open-metrics-explorer-0')).toBeInTheDocument();
expect(
screen.queryByTestId('open-metrics-explorer-1'),
).not.toBeInTheDocument();
});
it('builds metrics explorer link with relativeTime when a relative interval is selected', () => {
renderEntityMetrics({ selectedInterval: '5m' as Time });
const href = screen
.getByTestId('open-metrics-explorer-0')
.getAttribute('href');
expect(href).toContain('relativeTime=5m');
expect(href).not.toContain('startTime=');
expect(href).not.toContain('endTime=');
});
it('builds metrics explorer link with absolute time range in milliseconds for custom interval', () => {
renderEntityMetrics({ selectedInterval: 'custom' as Time });
const href = screen
.getByTestId('open-metrics-explorer-0')
.getAttribute('href');
expect(href).toContain(`startTime=${mockTimeRange.startTime * 1000}`);
expect(href).toContain(`endTime=${mockTimeRange.endTime * 1000}`);
expect(href).not.toContain('relativeTime=');
});
it('passes correct parameters to useEntityMetrics hook', () => {
renderEntityMetrics();
expect(mockUseEntityMetrics).toHaveBeenCalledWith(

View File

@@ -1,6 +1,8 @@
import { QueryParams } from 'constants/query';
import { PANEL_TYPES } from 'constants/queryBuilder';
import ROUTES from 'constants/routes';
import { ExplorerViews } from 'pages/LogsExplorer/utils';
import { Query } from 'types/api/queryBuilder/queryBuilderData';
// Mapping between panel types and explorer views
export const panelTypeToExplorerView: Record<PANEL_TYPES, ExplorerViews> = {
@@ -50,3 +52,36 @@ export const getExplorerViewFromUrl = (
export const getExplorerViewForPanelType = (
panelType: PANEL_TYPES,
): ExplorerViews => panelTypeToExplorerView[panelType];
export interface MetricsExplorerUrlParams {
query: Query;
relativeTime?: string;
startTimeMs?: number;
endTimeMs?: number;
}
export const getMetricsExplorerUrl = ({
query,
relativeTime,
startTimeMs,
endTimeMs,
}: MetricsExplorerUrlParams): string => {
const params = new URLSearchParams();
params.set(
QueryParams.compositeQuery,
encodeURIComponent(JSON.stringify(query)),
);
if (relativeTime) {
params.set(QueryParams.relativeTime, relativeTime);
} else {
if (startTimeMs !== undefined) {
params.set(QueryParams.startTime, String(startTimeMs));
}
if (endTimeMs !== undefined) {
params.set(QueryParams.endTime, String(endTimeMs));
}
}
return `${ROUTES.METRICS_EXPLORER_EXPLORER}?${params.toString()}`;
};

View File

@@ -4,26 +4,13 @@ import (
"net/http"
"github.com/SigNoz/signoz/pkg/http/handler"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/types"
"github.com/SigNoz/signoz/pkg/types/authtypes"
"github.com/SigNoz/signoz/pkg/types/coretypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/gorilla/mux"
)
func telemetryReadScopes() []string {
return []string{
coretypes.ResourceTelemetryResourceLogs.Scope(coretypes.VerbRead),
coretypes.ResourceTelemetryResourceTraces.Scope(coretypes.VerbRead),
coretypes.ResourceTelemetryResourceMetrics.Scope(coretypes.VerbRead),
coretypes.ResourceTelemetryResourceAuditLogs.Scope(coretypes.VerbRead),
coretypes.ResourceTelemetryResourceMeterMetrics.Scope(coretypes.VerbRead),
}
}
func (provider *provider) addQuerierRoutes(router *mux.Router) error {
if err := router.Handle("/api/v5/query_range", handler.New(provider.authzMiddleware.CheckResources(provider.querierHandler.QueryRange, authtypes.SigNozAdminRoleName, authtypes.SigNozEditorRoleName, authtypes.SigNozViewerRoleName), handler.OpenAPIDef{
if err := router.Handle("/api/v5/query_range", handler.New(provider.authzMiddleware.ViewAccess(provider.querierHandler.QueryRange), handler.OpenAPIDef{
ID: "QueryRangeV5",
Tags: []string{"querier"},
Summary: "Query range",
@@ -459,17 +446,12 @@ func (provider *provider) addQuerierRoutes(router *mux.Router) error {
ResponseContentType: "application/json",
SuccessStatusCode: http.StatusOK,
ErrorStatusCodes: []int{http.StatusBadRequest},
SecuritySchemes: newScopedSecuritySchemes(telemetryReadScopes()),
}, handler.WithResourceDefs(handler.TelemetryResourceDef{
Verb: coretypes.VerbRead,
Category: coretypes.ActionCategoryDataAccess,
Selector: querybuilder.TelemetrySelector,
Resources: querybuilder.QueryRangeResources,
}))).Methods(http.MethodPost).GetError(); err != nil {
SecuritySchemes: newSecuritySchemes(types.RoleViewer),
})).Methods(http.MethodPost).GetError(); err != nil {
return err
}
if err := router.Handle("/api/v5/query_range/preview", handler.New(provider.authzMiddleware.CheckResources(provider.querierHandler.QueryRangePreview, authtypes.SigNozAdminRoleName, authtypes.SigNozEditorRoleName, authtypes.SigNozViewerRoleName), handler.OpenAPIDef{
if err := router.Handle("/api/v5/query_range/preview", handler.New(provider.authzMiddleware.ViewAccess(provider.querierHandler.QueryRangePreview), handler.OpenAPIDef{
ID: "QueryRangePreviewV5",
Tags: []string{"querier"},
Summary: "Query range preview",
@@ -481,13 +463,8 @@ func (provider *provider) addQuerierRoutes(router *mux.Router) error {
ResponseContentType: "application/json",
SuccessStatusCode: http.StatusOK,
ErrorStatusCodes: []int{http.StatusBadRequest},
SecuritySchemes: newScopedSecuritySchemes(telemetryReadScopes()),
}, handler.WithResourceDefs(handler.TelemetryResourceDef{
Verb: coretypes.VerbRead,
Category: coretypes.ActionCategoryDataAccess,
Selector: querybuilder.TelemetrySelector,
Resources: querybuilder.QueryRangeResources,
}))).Methods(http.MethodPost).GetError(); err != nil {
SecuritySchemes: newSecuritySchemes(types.RoleViewer),
})).Methods(http.MethodPost).GetError(); err != nil {
return err
}

View File

@@ -1,9 +1,6 @@
package handler
import (
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types/coretypes"
)
import "github.com/SigNoz/signoz/pkg/types/coretypes"
type ResourceDef interface {
// resolveRequest is unexported to seal the interface. It returns a slice so a
@@ -100,31 +97,3 @@ func (def AttachDetachParentChildResourceDef) resolveRequest(ec coretypes.Extrac
),
}
}
type TelemetryResourceDef struct {
Verb coretypes.Verb
Category coretypes.ActionCategory
Selector coretypes.SelectorFunc
Resources coretypes.ResourceExtractor
}
func (def TelemetryResourceDef) resolveRequest(ec coretypes.ExtractorContext) []coretypes.ResolvedResource {
refs, err := def.Resources(ec)
if err != nil {
return []coretypes.ResolvedResource{coretypes.NewResolvedResourceWithError(def.Verb, def.Category, err)}
}
if len(refs) == 0 {
return []coretypes.ResolvedResource{coretypes.NewResolvedResourceWithError(
def.Verb,
def.Category,
errors.NewInvalidInputf(errors.CodeInvalidInput, "request resolved to no resources"),
)}
}
resolved := make([]coretypes.ResolvedResource, 0, len(refs))
for _, ref := range refs {
resolved = append(resolved, coretypes.NewResolvedResourceWithID(def.Verb, def.Category, ref.Resource, ref.ID, def.Selector))
}
return resolved
}

View File

@@ -118,10 +118,6 @@ func (middleware *Audit) emitAuditEvent(req *http.Request, writer responseCaptur
extractorCtx := coretypes.ExtractorContext{Request: req, ResponseBody: writer.BodyBytes()}
for _, resource := range resolved {
if err := resource.Err(); err != nil {
continue
}
resource.ResolveResponse(extractorCtx)
verb, category := resource.Verb(), resource.Category()

View File

@@ -1,239 +0,0 @@
package querybuilder
import (
"context"
"encoding/json"
"strings"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types/coretypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/tidwall/gjson"
)
func TelemetrySelector(_ context.Context, resource coretypes.Resource, id string, _ valuer.UUID) ([]coretypes.Selector, error) {
if coretypes.IsTelemetryQueryTypeSelector(id) {
id = id + "/" + coretypes.WildCardSelectorString
}
values := []string{id}
segments := strings.Split(id, "/")
for level := len(segments) - 1; level >= 1; level-- {
value := strings.Join(segments[:level], "/") + "/" + coretypes.WildCardSelectorString
if value == id {
continue
}
values = append(values, value)
}
if id != coretypes.WildCardSelectorString {
values = append(values, coretypes.WildCardSelectorString)
}
selectors := make([]coretypes.Selector, 0, len(values))
for _, value := range values {
selector, err := resource.Type().Selector(value)
if err != nil {
return nil, err
}
selectors = append(selectors, selector)
}
return selectors, nil
}
func QueryRangeResources(ec coretypes.ExtractorContext) ([]coretypes.ResourceWithID, error) {
queries := gjson.GetBytes(ec.RequestBody, "compositeQuery.queries")
if !queries.IsArray() || len(queries.Array()) == 0 {
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "composite query has no queries")
}
variables, err := queryRangeVariables(ec.RequestBody)
if err != nil {
return nil, err
}
refs := make([]coretypes.ResourceWithID, 0, len(queries.Array()))
seen := make(map[string]struct{})
for _, query := range queries.Array() {
queryRefs, err := resourcesForQuery(query, variables)
if err != nil {
return nil, err
}
for _, ref := range queryRefs {
key := ref.Resource.Kind().String() + ":" + ref.ID
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
refs = append(refs, ref)
}
}
return refs, nil
}
func queryRangeVariables(body []byte) (map[string]qbtypes.VariableItem, error) {
variables := make(map[string]qbtypes.VariableItem)
raw := gjson.GetBytes(body, "variables")
if !raw.Exists() {
return variables, nil
}
if err := json.Unmarshal([]byte(raw.Raw), &variables); err != nil {
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid variables in query range request")
}
return variables, nil
}
func resourcesForQuery(query gjson.Result, variables map[string]qbtypes.VariableItem) ([]coretypes.ResourceWithID, error) {
queryType := query.Get("type").String()
switch queryType {
case "builder_query", "builder_sub_query":
return resourcesForBuilderQuery(queryType, query.Get("spec"), variables)
case "builder_trace_operator":
return []coretypes.ResourceWithID{{Resource: coretypes.ResourceTelemetryResourceTraces, ID: queryType}}, nil
case "promql":
return []coretypes.ResourceWithID{{Resource: coretypes.ResourceTelemetryResourceMetrics, ID: queryType}}, nil
case "clickhouse_sql":
return []coretypes.ResourceWithID{
{Resource: coretypes.ResourceTelemetryResourceLogs, ID: queryType},
{Resource: coretypes.ResourceTelemetryResourceTraces, ID: queryType},
{Resource: coretypes.ResourceTelemetryResourceMetrics, ID: queryType},
}, nil
case "builder_formula", "builder_join":
return nil, nil
default:
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported query type %q", queryType)
}
}
func resourcesForBuilderQuery(queryType string, spec gjson.Result, variables map[string]qbtypes.VariableItem) ([]coretypes.ResourceWithID, error) {
resource, err := builderQueryResource(spec)
if err != nil {
return nil, err
}
ids, err := builderQuerySelectors(queryType, spec.Get("filter.expression").String(), variables)
if err != nil {
return nil, err
}
refs := make([]coretypes.ResourceWithID, 0, len(ids))
for _, id := range ids {
refs = append(refs, coretypes.ResourceWithID{Resource: resource, ID: id})
}
return refs, nil
}
func builderQueryResource(spec gjson.Result) (coretypes.Resource, error) {
source := spec.Get("source").String()
switch spec.Get("signal").String() {
case telemetrytypes.SignalTraces.StringValue():
return coretypes.ResourceTelemetryResourceTraces, nil
case telemetrytypes.SignalLogs.StringValue():
if source == telemetrytypes.SourceAudit.StringValue() {
return coretypes.ResourceTelemetryResourceAuditLogs, nil
}
return coretypes.ResourceTelemetryResourceLogs, nil
case telemetrytypes.SignalMetrics.StringValue():
if source == telemetrytypes.SourceMeter.StringValue() {
return coretypes.ResourceTelemetryResourceMeterMetrics, nil
}
return coretypes.ResourceTelemetryResourceMetrics, nil
default:
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported signal %q", spec.Get("signal").String())
}
}
func builderQuerySelectors(queryType, expression string, variables map[string]qbtypes.VariableItem) ([]string, error) {
if strings.TrimSpace(expression) == "" {
return []string{queryType}, nil
}
normalized, err := NormalizeWhereClause(expression, variables)
if err != nil {
return nil, err
}
ids := make([]string, 0)
for _, condition := range normalized.Conditions {
if !condition.TopLevel {
continue
}
key, ok := canonicalTelemetryGrantKey(condition.Key)
if !ok {
continue
}
if condition.Operator == "=" || condition.Operator == "IN" {
for _, value := range condition.Values {
ids = append(ids, queryType+"/"+key+"/"+value)
}
}
}
if len(ids) == 0 {
return []string{queryType}, nil
}
return ids, nil
}
func canonicalTelemetryGrantKey(keyText string) (string, bool) {
fieldKey := telemetrytypes.GetFieldKeyFromKeyText(keyText)
if fieldKey.FieldContext != telemetrytypes.FieldContextUnspecified && fieldKey.FieldContext != telemetrytypes.FieldContextResource {
return "", false
}
if !coretypes.IsTelemetryGrantKey(fieldKey.Name) {
return "", false
}
return fieldKey.Name, true
}
func ValidateTelemetryGrantSelector(input string) (string, error) {
if input == coretypes.WildCardSelectorString {
return input, nil
}
parts := strings.SplitN(input, "/", 3)
if !coretypes.IsTelemetryQueryTypeSelector(parts[0]) {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "telemetry selector %q must start with a query type or be %q", input, coretypes.WildCardSelectorString)
}
if len(parts) == 1 {
return parts[0] + "/" + coretypes.WildCardSelectorString, nil
}
if len(parts) == 2 {
if parts[1] != coretypes.WildCardSelectorString {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "telemetry selector %q must be <query_type>/<key>/<value>, <query_type>/%s or %s", input, coretypes.WildCardSelectorString, coretypes.WildCardSelectorString)
}
return input, nil
}
if !coretypes.IsTelemetryGrantKey(parts[1]) {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "telemetry selector %q must use one of the supported keys: %s", input, strings.Join(coretypes.TelemetryGrantKeys(), ", "))
}
value := parts[2]
if value == coretypes.WildCardSelectorString {
return input, nil
}
if value == "" || strings.HasPrefix(value, "$") {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "telemetry selector %q must use a concrete non-empty value", input)
}
return input, nil
}

View File

@@ -1,202 +0,0 @@
package querybuilder
import (
"context"
"strings"
"testing"
"github.com/SigNoz/signoz/pkg/types/coretypes"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func builderQueryBody(signal, filterExpression string) string {
return `{"compositeQuery":{"queries":[{"type":"builder_query","spec":{"signal":"` + signal + `","filter":{"expression":"` + filterExpression + `"}}}]}}`
}
func TestQueryRangeResources(t *testing.T) {
testCases := []struct {
name string
body string
expected []coretypes.ResourceWithID
}{
{
name: "top level service equality",
body: builderQueryBody("logs", "service.name = 'checkout' AND status = 500"),
expected: []coretypes.ResourceWithID{
{Resource: coretypes.ResourceTelemetryResourceLogs, ID: "builder_query/service.name/checkout"},
},
},
{
name: "resource prefixed service key",
body: builderQueryBody("traces", "resource.service.name = 'checkout'"),
expected: []coretypes.ResourceWithID{
{Resource: coretypes.ResourceTelemetryResourceTraces, ID: "builder_query/service.name/checkout"},
},
},
{
name: "in atom requires every value",
body: builderQueryBody("logs", "service.name IN ('b', 'a')"),
expected: []coretypes.ResourceWithID{
{Resource: coretypes.ResourceTelemetryResourceLogs, ID: "builder_query/service.name/a"},
{Resource: coretypes.ResourceTelemetryResourceLogs, ID: "builder_query/service.name/b"},
},
},
{
name: "multiple equality atoms each require a grant",
body: builderQueryBody("logs", "service.name = 'b' AND service.name = 'a'"),
expected: []coretypes.ResourceWithID{
{Resource: coretypes.ResourceTelemetryResourceLogs, ID: "builder_query/service.name/a"},
{Resource: coretypes.ResourceTelemetryResourceLogs, ID: "builder_query/service.name/b"},
},
},
{
name: "no filter expression",
body: `{"compositeQuery":{"queries":[{"type":"builder_query","spec":{"signal":"logs"}}]}}`,
expected: []coretypes.ResourceWithID{
{Resource: coretypes.ResourceTelemetryResourceLogs, ID: "builder_query"},
},
},
{
name: "service atom under or does not qualify",
body: builderQueryBody("logs", "service.name = 'a' OR status = 500"),
expected: []coretypes.ResourceWithID{
{Resource: coretypes.ResourceTelemetryResourceLogs, ID: "builder_query"},
},
},
{
name: "negated service atom does not qualify",
body: builderQueryBody("logs", "NOT service.name = 'a'"),
expected: []coretypes.ResourceWithID{
{Resource: coretypes.ResourceTelemetryResourceLogs, ID: "builder_query"},
},
},
{
name: "service inequality does not qualify",
body: builderQueryBody("logs", "service.name != 'a'"),
expected: []coretypes.ResourceWithID{
{Resource: coretypes.ResourceTelemetryResourceLogs, ID: "builder_query"},
},
},
{
name: "audit source maps to audit logs resource",
body: `{"compositeQuery":{"queries":[{"type":"builder_query","spec":{"signal":"logs","source":"audit","filter":{"expression":"service.name = 'a'"}}}]}}`,
expected: []coretypes.ResourceWithID{
{Resource: coretypes.ResourceTelemetryResourceAuditLogs, ID: "builder_query/service.name/a"},
},
},
{
name: "promql is wildcard only",
body: `{"compositeQuery":{"queries":[{"type":"promql","spec":{"query":"up"}}]}}`,
expected: []coretypes.ResourceWithID{
{Resource: coretypes.ResourceTelemetryResourceMetrics, ID: "promql"},
},
},
{
name: "clickhouse sql covers all signals",
body: `{"compositeQuery":{"queries":[{"type":"clickhouse_sql","spec":{"query":"SELECT 1"}}]}}`,
expected: []coretypes.ResourceWithID{
{Resource: coretypes.ResourceTelemetryResourceLogs, ID: "clickhouse_sql"},
{Resource: coretypes.ResourceTelemetryResourceTraces, ID: "clickhouse_sql"},
{Resource: coretypes.ResourceTelemetryResourceMetrics, ID: "clickhouse_sql"},
},
},
{
name: "formula produces no resources",
body: `{"compositeQuery":{"queries":[{"type":"builder_formula","spec":{"expression":"A/B"}}]}}`,
expected: []coretypes.ResourceWithID{},
},
{
name: "variable substitution qualifies",
body: `{"variables":{"svc":{"value":"checkout"}},"compositeQuery":{"queries":[{"type":"builder_query","spec":{"signal":"logs","filter":{"expression":"service.name = $svc"}}}]}}`,
expected: []coretypes.ResourceWithID{
{Resource: coretypes.ResourceTelemetryResourceLogs, ID: "builder_query/service.name/checkout"},
},
},
{
name: "duplicate queries dedupe",
body: `{"compositeQuery":{"queries":[{"type":"builder_query","spec":{"signal":"logs","filter":{"expression":"service.name = 'a'"}}},{"type":"builder_query","spec":{"signal":"logs","filter":{"expression":"service.name='a'"}}}]}}`,
expected: []coretypes.ResourceWithID{
{Resource: coretypes.ResourceTelemetryResourceLogs, ID: "builder_query/service.name/a"},
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
refs, err := QueryRangeResources(coretypes.ExtractorContext{RequestBody: []byte(testCase.body)})
require.NoError(t, err)
assert.Equal(t, testCase.expected, refs)
})
}
}
func TestQueryRangeResourcesErrors(t *testing.T) {
bodies := []string{
`{"compositeQuery":{"queries":[]}}`,
`{}`,
builderQueryBody("logs", "service.name = "),
`{"compositeQuery":{"queries":[{"type":"builder_query","spec":{"signal":"unknown"}}]}}`,
`{"compositeQuery":{"queries":[{"type":"unknown_type"}]}}`,
}
for _, body := range bodies {
_, err := QueryRangeResources(coretypes.ExtractorContext{RequestBody: []byte(body)})
assert.Error(t, err, "body %s", body)
}
}
func TestTelemetrySelector(t *testing.T) {
orgID := valuer.GenerateUUID()
selectorValues := func(id string) []string {
selectors, err := TelemetrySelector(context.Background(), coretypes.ResourceTelemetryResourceLogs, id, orgID)
require.NoError(t, err)
values := make([]string, 0, len(selectors))
for _, selector := range selectors {
values = append(values, selector.String())
}
return values
}
assert.Equal(t, []string{"builder_query/service.name/a", "builder_query/service.name/*", "builder_query/*", "*"}, selectorValues("builder_query/service.name/a"))
assert.Equal(t, []string{"builder_query/*", "*"}, selectorValues("builder_query"))
assert.Equal(t, []string{"promql/*", "*"}, selectorValues("promql"))
_, err := TelemetrySelector(context.Background(), coretypes.ResourceTelemetryResourceLogs, strings.Repeat("a", 256), orgID)
assert.Error(t, err)
}
func TestValidateTelemetryGrantSelector(t *testing.T) {
valid := map[string]string{
"*": "*",
"builder_query": "builder_query/*",
"promql": "promql/*",
"clickhouse_sql": "clickhouse_sql/*",
"builder_query/*": "builder_query/*",
"builder_query/service.name/checkout": "builder_query/service.name/checkout",
"builder_query/service.name/*": "builder_query/service.name/*",
"builder_query/service.name/a/b": "builder_query/service.name/a/b",
}
for input, expected := range valid {
canonical, err := ValidateTelemetryGrantSelector(input)
require.NoError(t, err, "input %q", input)
assert.Equal(t, expected, canonical, "input %q", input)
}
invalid := []string{
"",
"checkout",
"service.name = 'checkout'",
"builder_query/checkout",
"builder_query/deployment.environment/qa",
"builder_query/service.name/",
"builder_query/service.name/$svc",
"unknown_type/service.name/checkout",
}
for _, input := range invalid {
_, err := ValidateTelemetryGrantSelector(input)
assert.Error(t, err, "input %q", input)
}
}

View File

@@ -1,558 +0,0 @@
package querybuilder
import (
"fmt"
"sort"
"strconv"
"strings"
"github.com/SigNoz/signoz/pkg/errors"
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/antlr4-go/antlr/v4"
)
const WhereClauseOperatorFullText = "FULLTEXT"
type NormalizedWhereClause struct {
Expression string
Conditions []WhereClauseCondition
}
type WhereClauseCondition struct {
Key string
Operator string
Values []string
Negated bool
TopLevel bool
}
type joinKind int
const (
joinKindNone joinKind = iota
joinKindAnd
joinKindOr
)
type normalizedPart struct {
text string
join joinKind
skipped bool
}
type normalizedValue struct {
text string
raw string
}
type whereClauseNormalizer struct {
variables map[string]qbtypes.VariableItem
conditions []WhereClauseCondition
negated bool
orDepth int
errors []string
}
func NormalizeWhereClause(expression string, variables map[string]qbtypes.VariableItem) (*NormalizedWhereClause, error) {
input := antlr.NewInputStream(expression)
lexer := grammar.NewFilterQueryLexer(input)
lexerErrorListener := NewErrorListener()
lexer.RemoveErrorListeners()
lexer.AddErrorListener(lexerErrorListener)
tokens := antlr.NewCommonTokenStream(lexer, 0)
parser := grammar.NewFilterQueryParser(tokens)
parserErrorListener := NewErrorListener()
parser.RemoveErrorListeners()
parser.AddErrorListener(parserErrorListener)
tree := parser.Query()
syntaxErrors := append(lexerErrorListener.SyntaxErrors, parserErrorListener.SyntaxErrors...)
if len(syntaxErrors) > 0 {
combinedErrors := errors.Newf(
errors.TypeInvalidInput,
errors.CodeInvalidInput,
"Found %d syntax errors while parsing the filter expression.",
len(syntaxErrors),
)
additionals := make([]string, 0, len(syntaxErrors))
for _, syntaxError := range syntaxErrors {
if syntaxError.Error() != "" {
additionals = append(additionals, syntaxError.Error())
}
}
return nil, combinedErrors.WithAdditional(additionals...).WithUrl(searchTroubleshootingGuideURL)
}
visitor := &whereClauseNormalizer{
variables: variables,
conditions: make([]WhereClauseCondition, 0),
}
part := visitor.visitQuery(tree)
if len(visitor.errors) > 0 {
combinedErrors := errors.Newf(
errors.TypeInvalidInput,
errors.CodeInvalidInput,
"Found %d errors while parsing the filter expression.",
len(visitor.errors),
)
return nil, combinedErrors.WithAdditional(visitor.errors...).WithUrl(searchTroubleshootingGuideURL)
}
if part.skipped {
return &NormalizedWhereClause{Expression: "", Conditions: make([]WhereClauseCondition, 0)}, nil
}
sort.Slice(visitor.conditions, func(i, j int) bool {
return visitor.conditions[i].sortKey() < visitor.conditions[j].sortKey()
})
return &NormalizedWhereClause{Expression: part.text, Conditions: visitor.conditions}, nil
}
func (condition WhereClauseCondition) sortKey() string {
return condition.Key + "|" + condition.Operator + "|" + strings.Join(condition.Values, ",") + "|" + strconv.FormatBool(condition.Negated) + "|" + strconv.FormatBool(condition.TopLevel)
}
func (visitor *whereClauseNormalizer) visitQuery(ctx grammar.IQueryContext) normalizedPart {
if ctx.Expression() == nil {
return normalizedPart{skipped: true}
}
return visitor.visitOrExpression(ctx.Expression().OrExpression())
}
func (visitor *whereClauseNormalizer) visitOrExpression(ctx grammar.IOrExpressionContext) normalizedPart {
andExpressions := ctx.AllAndExpression()
if len(andExpressions) > 1 {
visitor.orDepth++
defer func() { visitor.orDepth-- }()
}
parts := make([]normalizedPart, 0, len(andExpressions))
for _, andExpression := range andExpressions {
part := visitor.visitAndExpression(andExpression)
if part.skipped {
continue
}
parts = append(parts, part)
}
if len(parts) == 0 {
return normalizedPart{skipped: true}
}
parts = sortAndDedupeNormalizedParts(parts)
if len(parts) == 1 {
return parts[0]
}
texts := make([]string, len(parts))
for index, part := range parts {
texts[index] = part.text
}
return normalizedPart{text: strings.Join(texts, " OR "), join: joinKindOr}
}
func (visitor *whereClauseNormalizer) visitAndExpression(ctx grammar.IAndExpressionContext) normalizedPart {
unaryExpressions := ctx.AllUnaryExpression()
parts := make([]normalizedPart, 0, len(unaryExpressions))
for _, unaryExpression := range unaryExpressions {
part := visitor.visitUnaryExpression(unaryExpression)
if part.skipped {
continue
}
if part.join == joinKindOr {
part = normalizedPart{text: "(" + part.text + ")", join: joinKindNone}
}
parts = append(parts, part)
}
if len(parts) == 0 {
return normalizedPart{skipped: true}
}
parts = sortAndDedupeNormalizedParts(parts)
if len(parts) == 1 {
return parts[0]
}
texts := make([]string, len(parts))
for index, part := range parts {
texts[index] = part.text
}
return normalizedPart{text: strings.Join(texts, " AND "), join: joinKindAnd}
}
func (visitor *whereClauseNormalizer) visitUnaryExpression(ctx grammar.IUnaryExpressionContext) normalizedPart {
negated := ctx.NOT() != nil
if negated {
visitor.negated = !visitor.negated
}
part := visitor.visitPrimary(ctx.Primary())
if negated {
visitor.negated = !visitor.negated
if part.skipped {
return part
}
if part.join != joinKindNone {
return normalizedPart{text: "NOT (" + part.text + ")", join: joinKindNone}
}
return normalizedPart{text: "NOT " + part.text, join: joinKindNone}
}
return part
}
func (visitor *whereClauseNormalizer) visitPrimary(ctx grammar.IPrimaryContext) normalizedPart {
if ctx.OrExpression() != nil {
return visitor.visitOrExpression(ctx.OrExpression())
}
if ctx.Comparison() != nil {
return visitor.visitComparison(ctx.Comparison())
}
if ctx.FunctionCall() != nil {
return normalizedPart{text: visitor.visitFunctionCall(ctx.FunctionCall())}
}
if ctx.FullText() != nil {
return normalizedPart{text: visitor.visitFullText(ctx.FullText())}
}
if ctx.Key() != nil {
return normalizedPart{text: visitor.fullTextTerm(ctx.Key().GetText())}
}
if ctx.Value() != nil {
value := visitor.normalizeValue(ctx.Value())
return normalizedPart{text: visitor.fullTextTerm(value.raw)}
}
return normalizedPart{skipped: true}
}
func (visitor *whereClauseNormalizer) visitComparison(ctx grammar.IComparisonContext) normalizedPart {
key := normalizeKeyText(ctx.Key().GetText())
if ctx.EXISTS() != nil {
operator := "EXISTS"
if ctx.NOT() != nil {
operator = "NOT EXISTS"
}
visitor.appendCondition(key, operator, nil)
return normalizedPart{text: key + " " + operator}
}
if ctx.InClause() != nil {
return visitor.visitInComparison(key, "IN", visitor.visitInValues(ctx.InClause().ValueList(), ctx.InClause().Value()))
}
if ctx.NotInClause() != nil {
return visitor.visitInComparison(key, "NOT IN", visitor.visitInValues(ctx.NotInClause().ValueList(), ctx.NotInClause().Value()))
}
if ctx.BETWEEN() != nil {
operator := "BETWEEN"
if ctx.NOT() != nil {
operator = "NOT BETWEEN"
}
values := ctx.AllValue()
low := visitor.normalizeValue(values[0])
high := visitor.normalizeValue(values[1])
visitor.appendCondition(key, operator, []string{low.raw, high.raw})
return normalizedPart{text: key + " " + operator + " " + low.text + " AND " + high.text}
}
operator := ""
switch {
case ctx.EQUALS() != nil:
operator = "="
case ctx.NOT_EQUALS() != nil, ctx.NEQ() != nil:
operator = "!="
case ctx.LT() != nil:
operator = "<"
case ctx.LE() != nil:
operator = "<="
case ctx.GT() != nil:
operator = ">"
case ctx.GE() != nil:
operator = ">="
case ctx.LIKE() != nil:
operator = "LIKE"
case ctx.ILIKE() != nil:
operator = "ILIKE"
case ctx.REGEXP() != nil:
operator = "REGEXP"
case ctx.CONTAINS() != nil:
operator = "CONTAINS"
}
if ctx.NOT() != nil {
operator = "NOT " + operator
}
value, skipped := visitor.substituteScalarVariable(visitor.normalizeValue(ctx.AllValue()[0]))
if skipped {
return normalizedPart{skipped: true}
}
visitor.appendCondition(key, operator, []string{value.raw})
return normalizedPart{text: key + " " + operator + " " + value.text}
}
func (visitor *whereClauseNormalizer) visitInComparison(key, operator string, values []normalizedValue) normalizedPart {
values, skipped := visitor.substituteListVariable(values)
if skipped {
return normalizedPart{skipped: true}
}
sort.Slice(values, func(i, j int) bool { return values[i].text < values[j].text })
texts := make([]string, 0, len(values))
raws := make([]string, 0, len(values))
for index, value := range values {
if index > 0 && value.text == values[index-1].text {
continue
}
texts = append(texts, value.text)
raws = append(raws, value.raw)
}
visitor.appendCondition(key, operator, raws)
return normalizedPart{text: key + " " + operator + " (" + strings.Join(texts, ", ") + ")"}
}
func (visitor *whereClauseNormalizer) visitInValues(valueList grammar.IValueListContext, value grammar.IValueContext) []normalizedValue {
values := make([]normalizedValue, 0)
if valueList != nil {
for _, valueCtx := range valueList.AllValue() {
values = append(values, visitor.normalizeValue(valueCtx))
}
return values
}
return append(values, visitor.normalizeValue(value))
}
func (visitor *whereClauseNormalizer) visitFunctionCall(ctx grammar.IFunctionCallContext) string {
functionName := ""
switch {
case ctx.HAS() != nil:
functionName = "has"
case ctx.HASANY() != nil:
functionName = "hasAny"
case ctx.HASALL() != nil:
functionName = "hasAll"
case ctx.HASTOKEN() != nil:
functionName = "hasToken"
}
key := ""
texts := make([]string, 0)
raws := make([]string, 0)
for index, param := range ctx.FunctionParamList().AllFunctionParam() {
switch {
case param.Key() != nil:
keyText := normalizeKeyText(param.Key().GetText())
if index == 0 {
key = keyText
} else {
raws = append(raws, keyText)
}
texts = append(texts, keyText)
case param.Value() != nil:
value := visitor.normalizeValue(param.Value())
texts = append(texts, value.text)
raws = append(raws, value.raw)
case param.Array() != nil:
arrayText, arrayRaws := visitor.visitArray(param.Array())
texts = append(texts, arrayText)
raws = append(raws, arrayRaws...)
}
}
visitor.appendCondition(key, functionName, raws)
return functionName + "(" + strings.Join(texts, ", ") + ")"
}
func (visitor *whereClauseNormalizer) visitArray(ctx grammar.IArrayContext) (string, []string) {
texts := make([]string, 0)
raws := make([]string, 0)
for _, valueCtx := range ctx.ValueList().AllValue() {
value := visitor.normalizeValue(valueCtx)
texts = append(texts, value.text)
raws = append(raws, value.raw)
}
return "[" + strings.Join(texts, ", ") + "]", raws
}
func (visitor *whereClauseNormalizer) visitFullText(ctx grammar.IFullTextContext) string {
if ctx.QUOTED_TEXT() != nil {
return visitor.fullTextTerm(trimQuotes(ctx.QUOTED_TEXT().GetText()))
}
return visitor.fullTextTerm(ctx.FREETEXT().GetText())
}
func (visitor *whereClauseNormalizer) fullTextTerm(term string) string {
visitor.appendCondition("", WhereClauseOperatorFullText, []string{term})
return quoteValue(term)
}
func (visitor *whereClauseNormalizer) normalizeValue(ctx grammar.IValueContext) normalizedValue {
switch {
case ctx.QUOTED_TEXT() != nil:
raw := trimQuotes(ctx.QUOTED_TEXT().GetText())
return normalizedValue{text: quoteValue(raw), raw: raw}
case ctx.NUMBER() != nil:
text := ctx.NUMBER().GetText()
return normalizedValue{text: text, raw: text}
case ctx.BOOL() != nil:
text := strings.ToLower(ctx.BOOL().GetText())
return normalizedValue{text: text, raw: text}
default:
raw := ctx.KEY().GetText()
if strings.HasPrefix(raw, "$") {
return normalizedValue{text: raw, raw: raw}
}
return normalizedValue{text: quoteValue(raw), raw: raw}
}
}
func (visitor *whereClauseNormalizer) appendCondition(key, operator string, values []string) {
if values == nil {
values = make([]string, 0)
}
visitor.conditions = append(visitor.conditions, WhereClauseCondition{
Key: key,
Operator: operator,
Values: values,
Negated: visitor.negated,
TopLevel: visitor.orDepth == 0 && !visitor.negated,
})
}
func (visitor *whereClauseNormalizer) substituteScalarVariable(value normalizedValue) (normalizedValue, bool) {
variableItem, ok := visitor.resolveVariable(value.raw)
if !ok {
return value, false
}
if skipped := visitor.errIfSkippedOrEmpty(variableItem, value.raw); skipped {
return normalizedValue{}, true
}
switch variableValues := variableItem.Value.(type) {
case []any:
return formatVariableValue(variableValues[0]), false
case any:
return formatVariableValue(variableValues), false
}
return value, false
}
func (visitor *whereClauseNormalizer) substituteListVariable(values []normalizedValue) ([]normalizedValue, bool) {
if len(values) != 1 {
return values, false
}
variableItem, ok := visitor.resolveVariable(values[0].raw)
if !ok {
return values, false
}
if skipped := visitor.errIfSkippedOrEmpty(variableItem, values[0].raw); skipped {
return nil, true
}
switch variableValues := variableItem.Value.(type) {
case []any:
substituted := make([]normalizedValue, 0, len(variableValues))
for _, variableValue := range variableValues {
substituted = append(substituted, formatVariableValue(variableValue))
}
return substituted, false
case any:
return []normalizedValue{formatVariableValue(variableValues)}, false
}
return values, false
}
func (visitor *whereClauseNormalizer) errIfSkippedOrEmpty(variableItem qbtypes.VariableItem, raw string) bool {
if variableItem.Type == qbtypes.DynamicVariableType {
if allValue, ok := variableItem.Value.(string); ok && allValue == "__all__" {
return true
}
}
if variableValues, ok := variableItem.Value.([]any); ok && len(variableValues) == 0 {
visitor.errors = append(visitor.errors, fmt.Sprintf("malformed request payload: variable `%s` used in expression has an empty list value", strings.TrimPrefix(raw, "$")))
return true
}
return false
}
func (visitor *whereClauseNormalizer) resolveVariable(raw string) (qbtypes.VariableItem, bool) {
if len(visitor.variables) == 0 {
return qbtypes.VariableItem{}, false
}
variableItem, ok := visitor.variables[raw]
if !ok && len(raw) > 0 {
variableItem, ok = visitor.variables[raw[1:]]
}
return variableItem, ok
}
func formatVariableValue(value any) normalizedValue {
switch typed := value.(type) {
case string:
return normalizedValue{text: quoteValue(typed), raw: typed}
case bool:
text := strconv.FormatBool(typed)
return normalizedValue{text: text, raw: text}
default:
text := fmt.Sprintf("%v", typed)
return normalizedValue{text: text, raw: text}
}
}
func normalizeKeyText(keyText string) string {
fieldKey := telemetrytypes.GetFieldKeyFromKeyText(keyText)
return telemetrytypes.TelemetryFieldKeyToText(&fieldKey)
}
func quoteValue(value string) string {
escaped := strings.ReplaceAll(value, `\`, `\\`)
escaped = strings.ReplaceAll(escaped, `'`, `\'`)
return "'" + escaped + "'"
}
func sortAndDedupeNormalizedParts(parts []normalizedPart) []normalizedPart {
sort.Slice(parts, func(i, j int) bool { return parts[i].text < parts[j].text })
deduped := parts[:0]
for index, part := range parts {
if index > 0 && part.text == parts[index-1].text {
continue
}
deduped = append(deduped, part)
}
return deduped
}

View File

@@ -1,421 +0,0 @@
package querybuilder
import (
"testing"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNormalizeWhereClauseEquivalenceClasses(t *testing.T) {
testCases := []struct {
name string
expressions []string
expected string
}{
{
name: "spacing and keyword case",
expressions: []string{
"service.name = 'frontend' AND status = 200",
"service.name='frontend' and status=200",
"service.name = 'frontend' AND status = 200",
"service.name = frontend AND status = 200",
},
expected: "service.name = 'frontend' AND status = 200",
},
{
name: "operand order",
expressions: []string{
"a = 1 AND b = 2",
"b = 2 AND a = 1",
},
expected: "a = 1 AND b = 2",
},
{
name: "implicit and explicit AND",
expressions: []string{
"a = 1 b = 2",
"a = 1 AND b = 2",
},
expected: "a = 1 AND b = 2",
},
{
name: "quote styles",
expressions: []string{
`a = "frontend"`,
"a = 'frontend'",
},
expected: "a = 'frontend'",
},
{
name: "redundant parentheses",
expressions: []string{
"(a = 1)",
"a = 1",
"((a = 1))",
},
expected: "a = 1",
},
{
name: "in clause forms and value order",
expressions: []string{
"a IN (1, 2)",
"a IN [2, 1]",
"a in (2, 1, 1)",
},
expected: "a IN (1, 2)",
},
{
name: "single value in",
expressions: []string{
"a IN 1",
"a IN (1)",
"a IN [1]",
},
expected: "a IN (1)",
},
{
name: "operator aliases",
expressions: []string{
"a == 1",
"a = 1",
},
expected: "a = 1",
},
{
name: "not equals aliases",
expressions: []string{
"a <> 1",
"a != 1",
},
expected: "a != 1",
},
{
name: "duplicate siblings",
expressions: []string{
"a = 1 AND a = 1",
"a = 1",
},
expected: "a = 1",
},
{
name: "grouped or under and",
expressions: []string{
"a = 1 AND (b = 2 OR c = 3)",
"(c = 3 OR b = 2) AND a = 1",
},
expected: "(b = 2 OR c = 3) AND a = 1",
},
{
name: "exists spellings",
expressions: []string{
"service.name EXISTS",
"service.name exists",
"service.name EXIST",
},
expected: "service.name EXISTS",
},
{
name: "contains spellings",
expressions: []string{
"body CONTAINS 'error'",
"body contain 'error'",
},
expected: "body CONTAINS 'error'",
},
{
name: "full text term forms",
expressions: []string{
`"panic"`,
"'panic'",
"panic",
},
expected: "'panic'",
},
{
name: "not without parens",
expressions: []string{
"NOT a = 1",
"not (a = 1)",
},
expected: "NOT a = 1",
},
{
name: "not over grouped or",
expressions: []string{
"NOT (b = 2 OR a = 1)",
"not (a = 1 or b = 2)",
},
expected: "NOT (a = 1 OR b = 2)",
},
{
name: "function name case",
expressions: []string{
"HAS(tags, 'x')",
"has(tags, 'x')",
},
expected: "has(tags, 'x')",
},
{
name: "boolean case",
expressions: []string{
"a = TRUE",
"a = true",
},
expected: "a = true",
},
{
name: "between",
expressions: []string{
"duration BETWEEN 1 AND 10",
"duration between 1 and 10",
},
expected: "duration BETWEEN 1 AND 10",
},
{
name: "not in",
expressions: []string{
"a NOT IN (2, 1)",
"a not in [1, 2]",
},
expected: "a NOT IN (1, 2)",
},
{
name: "key with datatype annotation",
expressions: []string{
"resource.service.name:string = 'x'",
},
expected: "resource.service.name:string = 'x'",
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
for _, expression := range testCase.expressions {
canonical, err := NormalizeWhereClause(expression, nil)
require.NoError(t, err, "expression %q", expression)
assert.Equal(t, testCase.expected, canonical.Expression, "expression %q", expression)
}
})
}
}
func TestNormalizeWhereClauseNonEquivalence(t *testing.T) {
testCases := []struct {
name string
left string
right string
}{
{name: "different values", left: "a = 1", right: "a = 2"},
{name: "different keys", left: "a = 1", right: "b = 1"},
{name: "different operators", left: "a = 1", right: "a != 1"},
{name: "no semantic rewrite of not", left: "NOT a = 1", right: "a != 1"},
{name: "number literals as authored", left: "a = 1.0", right: "a = 1"},
{name: "between bounds are ordered", left: "a BETWEEN 1 AND 10", right: "a BETWEEN 10 AND 1"},
{name: "function params are ordered", left: "has(tags, 'x')", right: "has('x', tags)"},
{name: "and vs or", left: "a = 1 AND b = 2", right: "a = 1 OR b = 2"},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
left, err := NormalizeWhereClause(testCase.left, nil)
require.NoError(t, err)
right, err := NormalizeWhereClause(testCase.right, nil)
require.NoError(t, err)
assert.NotEqual(t, left.Expression, right.Expression)
})
}
}
func TestNormalizeWhereClauseAtoms(t *testing.T) {
canonical, err := NormalizeWhereClause("NOT (a = 1 OR b IN ('y', 'x')) AND service.name EXISTS AND hasAny(tags, ['p', 'q']) AND \"panic\"", nil)
require.NoError(t, err)
expected := []WhereClauseCondition{
{Key: "a", Operator: "=", Values: []string{"1"}, Negated: true},
{Key: "b", Operator: "IN", Values: []string{"x", "y"}, Negated: true},
{Key: "service.name", Operator: "EXISTS", Values: []string{}, Negated: false, TopLevel: true},
{Key: "tags", Operator: "hasAny", Values: []string{"p", "q"}, Negated: false, TopLevel: true},
{Key: "", Operator: WhereClauseOperatorFullText, Values: []string{"panic"}, Negated: false, TopLevel: true},
}
assert.ElementsMatch(t, expected, canonical.Conditions)
}
func TestNormalizeWhereClauseTopLevel(t *testing.T) {
testCases := []struct {
name string
expression string
expected map[string]bool
}{
{
name: "and siblings are top level",
expression: "service.name = 'a' AND status = 500",
expected: map[string]bool{"service.name": true, "status": true},
},
{
name: "or branches are not top level",
expression: "service.name = 'a' OR status = 500",
expected: map[string]bool{"service.name": false, "status": false},
},
{
name: "and sibling stays top level next to a grouped or",
expression: "service.name = 'a' AND (x = 1 OR y = 2)",
expected: map[string]bool{"service.name": true, "x": false, "y": false},
},
{
name: "parenthesized pure and group stays top level",
expression: "(service.name = 'a' AND b = 2) AND c = 3",
expected: map[string]bool{"service.name": true, "b": true, "c": true},
},
{
name: "negated condition is not top level",
expression: "NOT service.name = 'a' AND status = 500",
expected: map[string]bool{"service.name": false, "status": true},
},
{
name: "double negation restores top level",
expression: "NOT (NOT (service.name = 'a'))",
expected: map[string]bool{"service.name": true},
},
{
name: "in condition under and is top level",
expression: "service.name IN ('a', 'b') AND x = 1",
expected: map[string]bool{"service.name": true, "x": true},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
normalized, err := NormalizeWhereClause(testCase.expression, nil)
require.NoError(t, err)
actual := make(map[string]bool)
for _, condition := range normalized.Conditions {
actual[condition.Key] = condition.TopLevel
}
assert.Equal(t, testCase.expected, actual)
})
}
}
func TestNormalizeWhereClauseEscaping(t *testing.T) {
canonical, err := NormalizeWhereClause(`a = "it's fine"`, nil)
require.NoError(t, err)
assert.Equal(t, `a = 'it\'s fine'`, canonical.Expression)
require.Len(t, canonical.Conditions, 1)
assert.Equal(t, []string{"it's fine"}, canonical.Conditions[0].Values)
equivalent, err := NormalizeWhereClause(canonical.Expression, nil)
require.NoError(t, err)
assert.Equal(t, canonical.Expression, equivalent.Expression)
assert.Equal(t, canonical.Conditions, equivalent.Conditions)
}
func TestNormalizeWhereClauseSyntaxError(t *testing.T) {
_, err := NormalizeWhereClause("a = ", nil)
require.Error(t, err)
_, err = NormalizeWhereClause("AND a = 1", nil)
require.Error(t, err)
}
func TestNormalizeWhereClauseIdempotence(t *testing.T) {
expressions := []string{
"service.name='frontend' and (status = 500 or status=502) not retired k8s.pod.name exists",
"a IN [3, 1, 2] AND hasAll(tags, ['a', 'b']) AND body CONTAINS 'x'",
"duration BETWEEN 1 AND 10 OR duration > 100",
`msg = 'with \'escapes\' and "quotes"'`,
}
for _, expression := range expressions {
first, err := NormalizeWhereClause(expression, nil)
require.NoError(t, err, "expression %q", expression)
second, err := NormalizeWhereClause(first.Expression, nil)
require.NoError(t, err, "canonical output %q must re-parse", first.Expression)
assert.Equal(t, first.Expression, second.Expression, "canonicalization must be idempotent for %q", expression)
}
}
func TestNormalizeWhereClauseVariables(t *testing.T) {
variables := map[string]qbtypes.VariableItem{
"service": {Value: "frontend"},
"statuses": {Value: []any{float64(502), float64(500)}},
"env": {Type: qbtypes.DynamicVariableType, Value: "__all__"},
"limit": {Value: float64(100)},
}
testCases := []struct {
name string
expression string
expected string
}{
{
name: "scalar substitution",
expression: "service.name = $service",
expected: "service.name = 'frontend'",
},
{
name: "array substitution in IN is sorted",
expression: "status IN $statuses",
expected: "status IN (500, 502)",
},
{
name: "numeric substitution",
expression: "duration > $limit",
expected: "duration > 100",
},
{
name: "dynamic all prunes the condition",
expression: "a = 1 AND deployment.environment IN $env",
expected: "a = 1",
},
{
name: "unknown variable stays a token",
expression: "service.name = $unknown",
expected: "service.name = $unknown",
},
{
name: "substituted forms hash-equal to concrete forms",
expression: "status IN (502, 500) AND service.name = 'frontend'",
expected: "service.name = 'frontend' AND status IN (500, 502)",
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
normalized, err := NormalizeWhereClause(testCase.expression, variables)
require.NoError(t, err, "expression %q", testCase.expression)
assert.Equal(t, testCase.expected, normalized.Expression)
})
}
substituted, err := NormalizeWhereClause("service.name = $service AND status IN $statuses", variables)
require.NoError(t, err)
concrete, err := NormalizeWhereClause("status IN (500, 502) AND service.name = 'frontend'", nil)
require.NoError(t, err)
assert.Equal(t, concrete.Expression, substituted.Expression)
assert.Equal(t, concrete.Conditions, substituted.Conditions)
}
func TestNormalizeWhereClauseVariablesFullyPruned(t *testing.T) {
variables := map[string]qbtypes.VariableItem{
"env": {Type: qbtypes.DynamicVariableType, Value: "__all__"},
}
normalized, err := NormalizeWhereClause("deployment.environment IN $env", variables)
require.NoError(t, err)
assert.Equal(t, "", normalized.Expression)
assert.Empty(t, normalized.Conditions)
}
func TestNormalizeWhereClauseVariablesEmptyList(t *testing.T) {
variables := map[string]qbtypes.VariableItem{
"statuses": {Value: []any{}},
}
_, err := NormalizeWhereClause("status IN $statuses", variables)
require.Error(t, err)
}

View File

@@ -729,7 +729,11 @@ func (v *filterExpressionVisitor) VisitFunctionCall(ctx *grammar.FunctionCallCon
return ErrorConditionLiteral
}
value := params[1:]
value, err := normalizeFunctionValue(operator, functionName, params[1:])
if err != nil {
v.errors = append(v.errors, err.Error())
return ErrorConditionLiteral
}
conds, ok := v.buildConditions(key, matchingFieldKeys(key, v.fieldKeys), operator, value)
if !ok {
@@ -745,6 +749,44 @@ func (v *filterExpressionVisitor) VisitFunctionCall(ctx *grammar.FunctionCallCon
return v.builder.Or(conds...)
}
// normalizeFunctionValue validates and normalizes the value argument(s) of a has-family
// function call, returning them in the wrapper slice the condition builder unwraps.
//
// - has/hasToken take exactly one scalar value. More than one argument, or an array
// argument, is rejected rather than silently dropping the extras.
// - hasAny/hasAll take a set of values, supplied either as a single array literal
// (hasAny(k, ['a','b'])) or as several scalar arguments (hasAny(k, 'a', 'b')); the
// latter are folded into one list so no argument is silently ignored.
func normalizeFunctionValue(operator qbtypes.FilterOperator, functionName string, valueParams []any) (any, error) {
switch operator {
case qbtypes.FilterOperatorHas, qbtypes.FilterOperatorHasToken:
if len(valueParams) != 1 {
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "function `%s` expects exactly one value argument", functionName)
}
if _, isArray := valueParams[0].([]any); isArray {
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "function `%s` expects a single scalar value, not an array", functionName)
}
return valueParams, nil
case qbtypes.FilterOperatorHasAny, qbtypes.FilterOperatorHasAll:
// A single array literal is already the value set.
if len(valueParams) == 1 {
if _, isArray := valueParams[0].([]any); isArray {
return valueParams, nil
}
}
// Otherwise fold the positional scalar arguments into one list.
values := make([]any, 0, len(valueParams))
for _, p := range valueParams {
if _, isArray := p.([]any); isArray {
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "function `%s` expects either a single array literal or scalar values, not a mix of the two", functionName)
}
values = append(values, p)
}
return []any{values}, nil
}
return valueParams, nil
}
// VisitFunctionParamList handles the parameter list for function calls.
func (v *filterExpressionVisitor) VisitFunctionParamList(ctx *grammar.FunctionParamListContext) any {
params := ctx.AllFunctionParam()

View File

@@ -40,12 +40,10 @@ func isBodyJSONSearch(key *telemetrytypes.TelemetryFieldKey, columns []*schema.C
return false
}
// conditionForArrayFunction builds `has/hasAny/hasAll(<arrayFieldExpr>, value)` over a
// body JSON array field. The field expression uses the JSON accessor (flag on) or
// legacy string extraction (flag off); value[0] is the needle.
// conditionForArrayFunction builds has/hasAny/hasAll over a body JSON path — via the JSON
// access plan (flag on) or legacy typed extraction (flag off).
func (c *conditionBuilder) conditionForArrayFunction(
ctx context.Context,
startNs, endNs uint64,
key *telemetrytypes.TelemetryFieldKey,
operator qbtypes.FilterOperator,
value any,
@@ -62,24 +60,48 @@ func (c *conditionBuilder) conditionForArrayFunction(
needle = args[0]
}
var fieldExpr string
if c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(valuer.UUID{})) {
fe, err := c.fm.FieldFor(ctx, startNs, endNs, key)
if err != nil {
return "", err
}
fieldExpr = fe
} else {
// legacy string-body path; value drives array-type inference (e.g. `[*]` paths)
fieldExpr, _ = GetBodyJSONKey(ctx, key, qbtypes.FilterOperatorUnknown, value)
// JSON access plan: data-type collision handling, nested array paths.
valueType, needle := InferDataType(needle, operator, key)
return NewJSONConditionBuilder(key, valueType).buildArrayFunctionCondition(operator, needle, sb)
}
return fmt.Sprintf("%s(%s, %s)", operator.FunctionName(), fieldExpr, sb.Var(needle)), nil
// legacy string-body path: type-matched array extraction, OR-ed with a scalar comparison
// for a scalar body value (coalesced to false so NOT has() matches missing-key rows).
elemType := legacyElemType(needle)
arrayExpr := getBodyJSONArrayKey(key, elemType)
scalarExpr, scalarGuard, hasScalar := getBodyJSONScalarKey(key, elemType)
if list, ok := needle.([]any); ok {
vals := make([]any, len(list))
for i, v := range list {
vals[i] = legacyCoerceNeedle(v, elemType)
}
arrayCond := fmt.Sprintf("%s(%s, %s)", operator.FunctionName(), arrayExpr, sb.Var(vals))
if !hasScalar {
return arrayCond, nil
}
var membership string
if operator == qbtypes.FilterOperatorHasAll {
eqs := make([]string, len(vals))
for i, v := range vals {
eqs[i] = sb.E(scalarExpr, v)
}
membership = sb.And(eqs...)
} else {
membership = sb.In(scalarExpr, vals...)
}
return fmt.Sprintf("(%s OR ifNull(%s, false))", arrayCond, sb.And(membership, scalarGuard)), nil
}
typedNeedle := legacyCoerceNeedle(needle, elemType)
arrayCond := fmt.Sprintf("%s(%s, %s)", operator.FunctionName(), arrayExpr, sb.Var(typedNeedle))
if !hasScalar {
return arrayCond, nil
}
return fmt.Sprintf("(%s OR ifNull(%s, false))", arrayCond, sb.And(sb.E(scalarExpr, typedNeedle), scalarGuard)), nil
}
// conditionForHasToken builds `hasToken(LOWER(<bodyColumn>), LOWER(<needle>))`, a
// full-text token search over the body column. It resolves the column from the key
// name + use_json_body flag, validates the field/value, and tags errors with the doc URL.
// conditionForHasToken builds a hasToken full-text search over the body column, resolving the
// column from the key name + use_json_body flag.
func (c *conditionBuilder) conditionForHasToken(
ctx context.Context,
key *telemetrytypes.TelemetryFieldKey,
@@ -92,28 +114,37 @@ func (c *conditionBuilder) conditionForHasToken(
needle = args[0]
}
// TODO(Tushar): thread orgID here to evaluate correctly
bodyJSONEnabled := c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(valuer.UUID{}))
columnName := LogsV2BodyColumn
if bodyJSONEnabled {
if key.Name != LogsV2BodyColumn && key.Name != bodyMessageField {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput,
"function `hasToken` only supports body/body.message field as first parameter").WithUrl(hasTokenFunctionDocURL)
}
columnName = bodyMessageField
} else if key.Name != LogsV2BodyColumn {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput,
"function `hasToken` only supports body field as first parameter").WithUrl(hasTokenFunctionDocURL)
}
// hasToken matches string tokens only.
if _, ok := needle.(string); !ok {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput,
"function `hasToken` expects value parameter to be a string").WithUrl(hasTokenFunctionDocURL)
}
return fmt.Sprintf("hasToken(LOWER(%s), LOWER(%s))", columnName, sb.Var(needle)), nil
// TODO(Tushar): thread orgID here to evaluate correctly
bodyJSONEnabled := c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(valuer.UUID{}))
if !bodyJSONEnabled {
// legacy: token search over the plain body string column only.
if key.Name != LogsV2BodyColumn {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput,
"function `hasToken` only supports body field as first parameter").WithUrl(hasTokenFunctionDocURL)
}
return fmt.Sprintf("hasToken(LOWER(%s), LOWER(%s))", LogsV2BodyColumn, sb.Var(needle)), nil
}
// JSON mode: a bare body/body.message key searches the body.message column; any other body
// field is a token search over its JSON string field, incl. strings nested in arrays.
// `body.message` resolves to a body-context key named `message`, so match that too — else it
// falls through and emits dynamicElement over the already-typed String column, which errors.
if key.Name == LogsV2BodyColumn || key.Name == bodyMessageField ||
(key.FieldContext == telemetrytypes.FieldContextBody && key.Name == messageSubField) {
return fmt.Sprintf("hasToken(LOWER(%s), LOWER(%s))", bodyMessageField, sb.Var(needle)), nil
}
if key.FieldContext == telemetrytypes.FieldContextBody {
return NewJSONConditionBuilder(key, telemetrytypes.FieldDataTypeString).buildTokenFunctionCondition(needle, sb)
}
return "", errors.NewInvalidInputf(errors.CodeInvalidInput,
"function `hasToken` only supports the body field or a body JSON string field as first parameter").WithUrl(hasTokenFunctionDocURL)
}
func (c *conditionBuilder) conditionFor(
@@ -124,8 +155,7 @@ func (c *conditionBuilder) conditionFor(
value any,
sb *sqlbuilder.SelectBuilder,
) (string, error) {
// hasToken is a token search over the body column resolved purely from the key
// name + flag, independent of column resolution, so handle it before anything else.
// hasToken resolves from the key name + flag alone (no column resolution), so handle it first.
if operator == qbtypes.FilterOperatorHasToken {
return c.conditionForHasToken(ctx, key, value, sb)
}
@@ -135,10 +165,9 @@ func (c *conditionBuilder) conditionFor(
return "", err
}
// has/hasAny/hasAll build `has(<arrayFieldExpr>, value)` over body JSON arrays
// rather than going through the normal operator paths, so handle them up front.
// has/hasAny/hasAll take the body-JSON path, not the normal operator paths.
if operator.IsArrayFunctionOperator() {
return c.conditionForArrayFunction(ctx, startNs, endNs, key, operator, value, columns, sb)
return c.conditionForArrayFunction(ctx, key, operator, value, columns, sb)
}
// TODO(Piyush): Update this to support multiple JSON columns based on evolutions
@@ -402,23 +431,6 @@ func (c *conditionBuilder) ConditionFor(
}
}
// has/hasAny/hasAll need an array field: in JSON-body mode drop non-array matches so a
// scalar errors clearly instead of failing at ClickHouse runtime (legacy mode skips this).
if operator.IsArrayFunctionOperator() &&
c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(valuer.UUID{})) {
arrayKeys := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys))
for _, k := range keys {
if k.FieldDataType.IsArray() {
arrayKeys = append(arrayKeys, k)
}
}
if len(arrayKeys) == 0 {
return nil, warnings, errors.NewInvalidInputf(errors.CodeInvalidInput,
"function `%s` expects key parameter to be an array field; no array fields found", operator.FunctionName())
}
keys = arrayKeys
}
conds := make([]string, 0, len(keys))
for _, k := range keys {
cond, err := c.conditionForKey(ctx, startNs, endNs, k, operator, value, sb)

View File

@@ -44,34 +44,66 @@ func TestFilterExprLogsBodyJSON(t *testing.T) {
category: "json",
query: "has(body.requestor_list[*], 'index_service')",
shouldPass: true,
expectedQuery: `WHERE has(JSONExtract(JSON_QUERY(body, '$."requestor_list"[*]'), 'Array(String)'), ?)`,
expectedArgs: []any{"index_service"},
expectedQuery: `WHERE (has(JSONExtract(JSON_QUERY(body, '$."requestor_list"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$."requestor_list"') = ? AND JSONType(body, 'requestor_list') NOT IN ('Array', 'Object', 'Null')), false))`,
expectedArgs: []any{"index_service", "index_service"},
expectedErrorContains: "",
},
{
category: "json",
query: "has(body.int_numbers[*], 2)",
shouldPass: true,
expectedQuery: `WHERE has(JSONExtract(JSON_QUERY(body, '$."int_numbers"[*]'), 'Array(Float64)'), ?)`,
expectedArgs: []any{float64(2)},
expectedQuery: `WHERE (has(JSONExtract(JSON_QUERY(body, '$."int_numbers"[*]'), 'Array(Nullable(Float64))'), ?) OR ifNull((JSONExtract(JSON_VALUE(body, '$."int_numbers"'), 'Nullable(Float64)') = ? AND JSONType(body, 'int_numbers') NOT IN ('Array', 'Object', 'Null')), false))`,
expectedArgs: []any{float64(2), float64(2)},
expectedErrorContains: "",
},
{
category: "json",
query: "has(body.bool[*], true)",
shouldPass: true,
expectedQuery: `WHERE has(JSONExtract(JSON_QUERY(body, '$."bool"[*]'), 'Array(Bool)'), ?)`,
expectedArgs: []any{true},
expectedQuery: `WHERE (has(JSONExtract(JSON_QUERY(body, '$."bool"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$."bool"') = ? AND JSONType(body, 'bool') NOT IN ('Array', 'Object', 'Null')), false))`,
expectedArgs: []any{"true", "true"},
expectedErrorContains: "",
},
{
category: "json",
query: "NOT has(body.nested_num[*].float_nums[*], 2.2)",
shouldPass: true,
expectedQuery: `WHERE NOT (has(JSONExtract(JSON_QUERY(body, '$."nested_num"[*]."float_nums"[*]'), 'Array(Float64)'), ?))`,
expectedQuery: `WHERE NOT (has(JSONExtract(JSON_QUERY(body, '$."nested_num"[*]."float_nums"[*]'), 'Array(Nullable(Float64))'), ?))`,
expectedArgs: []any{float64(2.2)},
expectedErrorContains: "",
},
{
category: "json",
query: "has(body.tags, 'production')",
shouldPass: true,
expectedQuery: `WHERE (has(JSONExtract(JSON_QUERY(body, '$."tags"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$."tags"') = ? AND JSONType(body, 'tags') NOT IN ('Array', 'Object', 'Null')), false))`,
expectedArgs: []any{"production", "production"},
expectedErrorContains: "",
},
{
category: "json",
query: "hasAny(body.tags, ['critical', 'test'])",
shouldPass: true,
expectedQuery: `WHERE (hasAny(JSONExtract(JSON_QUERY(body, '$."tags"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$."tags"') IN (?, ?) AND JSONType(body, 'tags') NOT IN ('Array', 'Object', 'Null')), false))`,
expectedArgs: []any{[]any{"critical", "test"}, "critical", "test"},
expectedErrorContains: "",
},
{
category: "json",
query: "hasAll(body.tags, ['production', 'web'])",
shouldPass: true,
expectedQuery: `WHERE (hasAll(JSONExtract(JSON_QUERY(body, '$."tags"[*]'), 'Array(Nullable(String))'), ?) OR ifNull(((JSON_VALUE(body, '$."tags"') = ? AND JSON_VALUE(body, '$."tags"') = ?) AND JSONType(body, 'tags') NOT IN ('Array', 'Object', 'Null')), false))`,
expectedArgs: []any{[]any{"production", "web"}, "production", "web"},
expectedErrorContains: "",
},
{
category: "json",
query: "has(body.ids, \"200\")",
shouldPass: true,
expectedQuery: `WHERE (has(JSONExtract(JSON_QUERY(body, '$."ids"[*]'), 'Array(Nullable(Int64))'), ?) OR ifNull((JSONExtract(JSON_VALUE(body, '$."ids"'), 'Nullable(Int64)') = ? AND JSONType(body, 'ids') NOT IN ('Array', 'Object', 'Null')), false))`,
expectedArgs: []any{int64(200), int64(200)},
expectedErrorContains: "",
},
{
category: "json",
query: "body.message = hello",

View File

@@ -1561,6 +1561,25 @@ func TestFilterExprLogs(t *testing.T) {
expectedArgs: []any{"download"},
expectedErrorContains: "function `hasToken` expects value parameter to be a string",
},
// extra / mis-shaped value arguments are rejected, not silently dropped.
{
category: "hasExtraArgs",
query: "has(body.tags[*], \"a\", \"b\")",
shouldPass: false,
expectedErrorContains: "function `has` expects exactly one value argument",
},
{
category: "hasArrayArg",
query: "has(body.tags[*], [\"a\", \"b\"])",
shouldPass: false,
expectedErrorContains: "function `has` expects a single scalar value, not an array",
},
{
category: "hasTokenExtraArgs",
query: "hasToken(body, \"a\", \"b\")",
shouldPass: false,
expectedErrorContains: "function `hasToken` expects exactly one value argument",
},
// Basic materialized key
{

View File

@@ -96,6 +96,18 @@ func applyNotCondition(operator qbtypes.FilterOperator) (bool, qbtypes.FilterOpe
return false, operator
}
// branchArrayExpr returns the ClickHouse array expression for a given array-type branch
// at this hop. The JSON branch reads Array(JSON(...)) directly; the Dynamic branch filters
// the Array(Dynamic) down to its JSON elements and maps them to JSON.
func (c *jsonConditionBuilder) branchArrayExpr(node *telemetrytypes.JSONAccessNode, branch telemetrytypes.JSONAccessBranchType) string {
fieldPath := node.FieldPath()
if branch == telemetrytypes.BranchDynamic {
dynBaseExpr := fmt.Sprintf("dynamicElement(%s, 'Array(Dynamic)')", fieldPath)
return fmt.Sprintf("arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), %s))", dynBaseExpr)
}
return fmt.Sprintf("dynamicElement(%s, 'Array(JSON(max_dynamic_types=%d, max_dynamic_paths=%d))')", fieldPath, node.MaxDynamicTypes, node.MaxDynamicPaths)
}
// buildAccessNodeBranches builds conditions for each branch of the access node.
func (c *jsonConditionBuilder) buildAccessNodeBranches(current *telemetrytypes.JSONAccessNode, operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error) {
if current == nil {
@@ -103,31 +115,15 @@ func (c *jsonConditionBuilder) buildAccessNodeBranches(current *telemetrytypes.J
}
currAlias := current.Alias()
fieldPath := current.FieldPath()
// Determine availability of Array(JSON) and Array(Dynamic) at this hop
hasArrayJSON := current.Branches[telemetrytypes.BranchJSON] != nil
hasArrayDynamic := current.Branches[telemetrytypes.BranchDynamic] != nil
// Then, at this hop, compute child per branch and wrap
// At this hop, compute the child condition per array branch (JSON before Dynamic) and
// wrap each in arrayExists over the corresponding array expression.
branches := make([]string, 0, 2)
if hasArrayJSON {
jsonArrayExpr := fmt.Sprintf("dynamicElement(%s, 'Array(JSON(max_dynamic_types=%d, max_dynamic_paths=%d))')", fieldPath, current.MaxDynamicTypes, current.MaxDynamicPaths)
childGroupJSON, err := c.recurseArrayHops(current.Branches[telemetrytypes.BranchJSON], operator, value, sb)
for _, branch := range current.BranchesInOrder() {
childGroup, err := c.recurseArrayHops(current.Branches[branch], operator, value, sb)
if err != nil {
return "", err
}
branches = append(branches, fmt.Sprintf("arrayExists(%s-> %s, %s)", currAlias, childGroupJSON, jsonArrayExpr))
}
if hasArrayDynamic {
dynBaseExpr := fmt.Sprintf("dynamicElement(%s, 'Array(Dynamic)')", fieldPath)
dynFilteredExpr := fmt.Sprintf("arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), %s))", dynBaseExpr)
// Create the Query for Dynamic array
childGroupDyn, err := c.recurseArrayHops(current.Branches[telemetrytypes.BranchDynamic], operator, value, sb)
if err != nil {
return "", err
}
branches = append(branches, fmt.Sprintf("arrayExists(%s-> %s, %s)", currAlias, childGroupDyn, dynFilteredExpr))
branches = append(branches, fmt.Sprintf("arrayExists(%s-> %s, %s)", currAlias, childGroup, c.branchArrayExpr(current, branch)))
}
if len(branches) == 1 {
@@ -309,6 +305,174 @@ func (c *jsonConditionBuilder) buildArrayMembershipCondition(node *telemetrytype
return fmt.Sprintf("arrayExists(%s -> %s, %s)", key, op, arrayExpr), nil
}
// buildArrayFunctionCondition builds a has/hasAny/hasAll condition over a body JSON path,
// with contains-all semantics uniform across every leaf shape:
// - has(v) = the path HAS v
// - hasAny([v...]) = the path has ANY listed value (OR of has)
// - hasAll([v...]) = the path has ALL listed values (AND of has)
//
// "the path has v" is an existential match resolved per leaf shape: for an array-typed leaf
// (top-level `body.tags`, or nested `body.education[].scores`) it is native membership; for a
// scalar leaf — whether reached through an array hop (`body.items[].sku`) or a plain scalar
// path (`body.level`) — it is `<elem> = v`, wrapped in arrayExists over any array hops. So
// `hasAll(body.education[].name, ['a','b'])` = "some element is a AND some element is b", and
// for a plain scalar hasAll collapses to has (a one-element set can hold at most one value).
//
// Element comparisons reuse DataTypeCollisionHandledFieldName so a numeric literal against an
// Int64 array (or a numeric literal against a String array) no longer silently misses.
func (c *jsonConditionBuilder) buildArrayFunctionCondition(operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error) {
if len(c.key.JSONPlan) == 0 {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "function `%s` could not resolve a JSON access plan for field `%s`", operator.FunctionName(), c.key.Name)
}
switch operator {
case qbtypes.FilterOperatorHas, qbtypes.FilterOperatorHasAny:
return c.buildOredRootChains(func(node *telemetrytypes.JSONAccessNode) (string, error) {
return c.arrayFunctionLeaf(node, operator, value, sb)
}, sb)
case qbtypes.FilterOperatorHasAll:
// contains-all: AND of a per-value "has" so the AND sits outside the array hops.
values := toAnyList(value)
conditions := make([]string, 0, len(values))
for _, v := range values {
v := v
cond, err := c.buildOredRootChains(func(node *telemetrytypes.JSONAccessNode) (string, error) {
return c.arrayFunctionLeaf(node, qbtypes.FilterOperatorHas, v, sb)
}, sb)
if err != nil {
return "", err
}
conditions = append(conditions, cond)
}
if len(conditions) == 1 {
return conditions[0], nil
}
return sb.And(conditions...), nil
}
return "", qbtypes.ErrUnsupportedOperator
}
// buildOredRootChains applies leafFn down every JSONPlan root (base + promoted), wrapping each
// in its arrayExists chain, and ORs the per-root results.
func (c *jsonConditionBuilder) buildOredRootChains(leafFn func(*telemetrytypes.JSONAccessNode) (string, error), sb *sqlbuilder.SelectBuilder) (string, error) {
conditions := make([]string, 0, len(c.key.JSONPlan))
for _, root := range c.key.JSONPlan {
cond, err := c.buildArrayExistsChain(root, leafFn, sb)
if err != nil {
return "", err
}
conditions = append(conditions, cond)
}
if len(conditions) == 1 {
return conditions[0], nil
}
return sb.Or(conditions...), nil
}
// buildArrayExistsChain wraps the terminal condition (produced by leafFn) in an arrayExists
// over every array hop between the root and the terminal. For a terminal root (a top-level
// array leaf) it simply returns leafFn(root).
func (c *jsonConditionBuilder) buildArrayExistsChain(node *telemetrytypes.JSONAccessNode, leafFn func(*telemetrytypes.JSONAccessNode) (string, error), sb *sqlbuilder.SelectBuilder) (string, error) {
if node == nil {
return "", errors.NewInternalf(CodeArrayNavigationFailed, "navigation failed, current node is nil")
}
if node.IsTerminal {
return leafFn(node)
}
branches := make([]string, 0, 2)
for _, branch := range node.BranchesInOrder() {
childCond, err := c.buildArrayExistsChain(node.Branches[branch], leafFn, sb)
if err != nil {
return "", err
}
branches = append(branches, fmt.Sprintf("arrayExists(%s-> %s, %s)", node.Alias(), childCond, c.branchArrayExpr(node, branch)))
}
if len(branches) == 1 {
return branches[0], nil
}
return sb.Or(branches...), nil
}
// arrayFunctionLeaf builds the existential comparison for has/hasAny at a terminal node (hasAll
// composes from has in buildArrayFunctionCondition). For an array leaf it delegates to native
// membership; for a scalar leaf it compares the element directly.
func (c *jsonConditionBuilder) arrayFunctionLeaf(node *telemetrytypes.JSONAccessNode, operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error) {
if node.TerminalConfig.ElemType.IsArray {
return c.arrayLeafMembership(node, operator, value, sb)
}
switch operator {
case qbtypes.FilterOperatorHas:
return c.arrayFuncScalarLeaf(node, qbtypes.FilterOperatorEqual, value, sb)
case qbtypes.FilterOperatorHasAny:
return c.arrayFuncScalarLeaf(node, qbtypes.FilterOperatorIn, toAnyList(value), sb)
}
return "", qbtypes.ErrUnsupportedOperator
}
// arrayLeafMembership builds native membership for an array-typed leaf, reusing
// buildArrayMembershipCondition (which handles data-type collisions on each element).
func (c *jsonConditionBuilder) arrayLeafMembership(node *telemetrytypes.JSONAccessNode, operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error) {
switch operator {
case qbtypes.FilterOperatorHas:
return c.buildArrayMembershipCondition(node, qbtypes.FilterOperatorEqual, value, sb)
case qbtypes.FilterOperatorHasAny:
return c.buildArrayMembershipCondition(node, qbtypes.FilterOperatorIn, toAnyList(value), sb)
}
return "", qbtypes.ErrUnsupportedOperator
}
// arrayFuncScalarLeaf builds `<elemExpr> <op> value` for a scalar leaf reached through an
// array hop, applying data-type collision handling like the standard primitive path.
// Coalesced to false so a missing key is a non-match, not NULL (NOT has() must match it).
func (c *jsonConditionBuilder) arrayFuncScalarLeaf(node *telemetrytypes.JSONAccessNode, operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error) {
fieldExpr := fmt.Sprintf("dynamicElement(%s, '%s')", node.FieldPath(), node.TerminalConfig.ElemType.StringValue())
fieldExpr, value = querybuilder.DataTypeCollisionHandledFieldName(node.TerminalConfig.Key, value, fieldExpr, operator)
cond, err := c.applyOperator(sb, fieldExpr, operator, value)
if err != nil {
return "", err
}
return fmt.Sprintf("ifNull(%s, false)", cond), nil
}
// buildTokenFunctionCondition builds a hasToken search over a body JSON string field:
// hasToken(LOWER(<elem>), LOWER(?)) wrapped in arrayExists over any array hops between the
// root and the terminal. The field must resolve to a String leaf or a String array.
func (c *jsonConditionBuilder) buildTokenFunctionCondition(needle any, sb *sqlbuilder.SelectBuilder) (string, error) {
if len(c.key.JSONPlan) == 0 {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "function `hasToken` could not resolve a JSON access plan for field `%s`", c.key.Name)
}
return c.buildOredRootChains(func(node *telemetrytypes.JSONAccessNode) (string, error) {
return c.tokenLeaf(node, needle, sb)
}, sb)
}
// tokenLeaf builds the hasToken match at a terminal node: a direct match for a String leaf
// (coalesced to false, as in arrayFuncScalarLeaf), or an arrayExists over the elements for a
// String array leaf. hasToken is string-only, so any other element type is rejected.
func (c *jsonConditionBuilder) tokenLeaf(node *telemetrytypes.JSONAccessNode, needle any, sb *sqlbuilder.SelectBuilder) (string, error) {
switch node.TerminalConfig.ElemType {
case telemetrytypes.String:
fieldExpr := fmt.Sprintf("dynamicElement(%s, 'String')", node.FieldPath())
return fmt.Sprintf("ifNull(hasToken(LOWER(%s), LOWER(%s)), false)", fieldExpr, sb.Var(needle)), nil
case telemetrytypes.ArrayString:
arrayExpr := fmt.Sprintf("dynamicElement(%s, '%s')", node.FieldPath(), node.TerminalConfig.ElemType.StringValue())
return fmt.Sprintf("arrayExists(x -> hasToken(LOWER(x), LOWER(%s)), %s)", sb.Var(needle), arrayExpr), nil
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "function `hasToken` only supports string fields; field `%s` is `%s`", c.key.Name, node.TerminalConfig.Key.FieldDataType.StringValue())
}
}
// toAnyList normalizes a has-family value into a slice; a scalar becomes a one-element list.
func toAnyList(value any) []any {
if list, ok := value.([]any); ok {
return list
}
return []any{value}
}
func (c *jsonConditionBuilder) applyOperator(sb *sqlbuilder.SelectBuilder, fieldExpr string, operator qbtypes.FilterOperator, value any) (string, error) {
switch operator {
case qbtypes.FilterOperatorEqual:

View File

@@ -602,7 +602,7 @@ func TestJSONStmtBuilder_ArrayPaths(t *testing.T) {
name: "Simple has filter",
filter: "has(body.education[].parameters, 1.65)",
expected: TestExpected{
WhereClause: "(has(arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))), ?) OR has(arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))), ?))",
WhereClause: "(arrayExists(`body_v2.education`-> arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education`-> arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
Args: []any{1.65, 1.65, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
Warnings: []string{
"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64 name=education[].parameters,context=body,datatype=[]dynamic].",
@@ -613,8 +613,8 @@ func TestJSONStmtBuilder_ArrayPaths(t *testing.T) {
name: "Flat path hasAll filter",
filter: "hasAll(body.user.permissions, ['read', 'write'])",
expected: TestExpected{
WhereClause: "hasAll(dynamicElement(body_v2.`user.permissions`, 'Array(Nullable(String))'), ?)",
Args: []any{[]any{"read", "write"}, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
WhereClause: "(arrayExists(x -> x = ?, dynamicElement(body_v2.`user.permissions`, 'Array(Nullable(String))')) AND arrayExists(x -> x = ?, dynamicElement(body_v2.`user.permissions`, 'Array(Nullable(String))')))",
Args: []any{"read", "write", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
@@ -739,8 +739,8 @@ func TestJSONStmtBuilder_ArrayPaths(t *testing.T) {
name: "Nested path hasAny filter",
filter: "hasAny(education[].awards[].participated[].members, ['Piyush', 'Tushar'])",
expected: TestExpected{
WhereClause: "hasAny(arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))), ?)",
Args: []any{[]any{"Piyush", "Tushar"}, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
WhereClause: "arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> x IN (?, ?), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> x IN (?, ?), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> x IN (?, ?), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> x IN (?, ?), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
Args: []any{"Piyush", "Tushar", "Piyush", "Tushar", "Piyush", "Tushar", "Piyush", "Tushar", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
{
@@ -755,8 +755,8 @@ func TestJSONStmtBuilder_ArrayPaths(t *testing.T) {
name: "dynamic_array_element_compare_HAS_STRING",
filter: "has(interests[].entities[].product_codes, '2002')",
expected: TestExpected{
WhereClause: "has(arrayFlatten(arrayConcat(arrayMap(`body_v2.interests`->arrayMap(`body_v2.interests[].entities`->dynamicElement(`body_v2.interests[].entities`.`product_codes`, 'Array(Dynamic)'), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))), ?)",
Args: []any{"2002", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
WhereClause: "arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.interests[].entities`.`product_codes`, 'Array(Dynamic)'))), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
Args: []any{int64(2002), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
{
@@ -771,10 +771,146 @@ func TestJSONStmtBuilder_ArrayPaths(t *testing.T) {
name: "dynamic_array_element_compare_HAS_INT",
filter: "has(interests[].entities[].product_codes, 1001)",
expected: TestExpected{
WhereClause: "has(arrayFlatten(arrayConcat(arrayMap(`body_v2.interests`->arrayMap(`body_v2.interests[].entities`->dynamicElement(`body_v2.interests[].entities`.`product_codes`, 'Array(Dynamic)'), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))), ?)",
WhereClause: "arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.interests[].entities`.`product_codes`, 'Array(Dynamic)'))), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
Args: []any{float64(1001), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
// ── scalar leaf reached through an array ───
{
name: "Nested primitive leaf has",
filter: "has(body.education[].name, 'IIT')",
expected: TestExpected{
WhereClause: "arrayExists(`body_v2.education`-> ifNull(dynamicElement(`body_v2.education`.`name`, 'String') = ?, false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
Args: []any{"IIT", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
{
name: "Nested primitive leaf hasAny",
filter: "hasAny(body.education[].name, ['IIT', 'MIT'])",
expected: TestExpected{
WhereClause: "arrayExists(`body_v2.education`-> ifNull(dynamicElement(`body_v2.education`.`name`, 'String') IN (?, ?), false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
Args: []any{"IIT", "MIT", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
{
// contains-all: single-value hasAll over a nested leaf collapses to has.
name: "Nested primitive leaf hasAll single collapses to has",
filter: "hasAll(body.education[].name, 'IIT')",
expected: TestExpected{
WhereClause: "arrayExists(`body_v2.education`-> ifNull(dynamicElement(`body_v2.education`.`name`, 'String') = ?, false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
Args: []any{"IIT", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
{
// contains-all: "some element is IIT AND some element is MIT" (AND of per-value has).
name: "Nested primitive leaf hasAll multi (contains-all)",
filter: "hasAll(body.education[].name, ['IIT', 'MIT'])",
expected: TestExpected{
WhereClause: "(arrayExists(`body_v2.education`-> ifNull(dynamicElement(`body_v2.education`.`name`, 'String') = ?, false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) AND arrayExists(`body_v2.education`-> ifNull(dynamicElement(`body_v2.education`.`name`, 'String') = ?, false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
Args: []any{"IIT", "MIT", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
// ── numeric literal against an Int64 array is collision-handled ───
{
name: "Nested Int64 array has collision",
filter: "has(body.education[].scores, 90)",
expected: TestExpected{
WhereClause: "arrayExists(`body_v2.education`-> arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.education`.`scores`, 'Array(Nullable(Int64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
Args: []any{float64(90), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
// ── hasAny folds multiple scalar arguments into one value set ─────
{
name: "hasAny folds multiple scalar args",
filter: "hasAny(body.user.permissions, 'read', 'write')",
expected: TestExpected{
WhereClause: "arrayExists(x -> x IN (?, ?), dynamicElement(body_v2.`user.permissions`, 'Array(Nullable(String))'))",
Args: []any{"read", "write", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
// ── hasToken over JSON string fields (nested leaf, top-level array, nested array) ──
{
name: "hasToken nested string leaf",
filter: "hasToken(body.education[].name, 'harvard')",
expected: TestExpected{
WhereClause: "arrayExists(`body_v2.education`-> ifNull(hasToken(LOWER(dynamicElement(`body_v2.education`.`name`, 'String')), LOWER(?)), false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
Args: []any{"harvard", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
{
name: "hasToken top-level string array",
filter: "hasToken(body.user.permissions, 'admin')",
expected: TestExpected{
WhereClause: "arrayExists(x -> hasToken(LOWER(x), LOWER(?)), dynamicElement(body_v2.`user.permissions`, 'Array(Nullable(String))'))",
Args: []any{"admin", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
{
name: "hasToken nested string array",
filter: "hasToken(body.education[].awards[].participated[].members, 'piyush')",
expected: TestExpected{
WhereClause: "arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> hasToken(LOWER(x), LOWER(?)), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> hasToken(LOWER(x), LOWER(?)), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> hasToken(LOWER(x), LOWER(?)), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> hasToken(LOWER(x), LOWER(?)), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
Args: []any{"piyush", "piyush", "piyush", "piyush", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
// ── hasToken over the message field: bare body and explicit body.message are
// equivalent, both target the body.message column directly (no dynamicElement) ──
{
name: "hasToken bare body",
filter: "hasToken(body, 'production')",
expected: TestExpected{
WhereClause: "hasToken(LOWER(body.message), LOWER(?))",
Args: []any{"production", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
Warnings: []string{bodySearchDefaultWarning},
},
},
{
name: "hasToken explicit body.message",
filter: "hasToken(body.message, 'production')",
expected: TestExpected{
WhereClause: "hasToken(LOWER(body.message), LOWER(?))",
Args: []any{"production", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
// ── scalar (non-array) leaf: treated as a single-element set ─────────────
{
name: "Scalar leaf has",
filter: "has(body.user.name, 'alice')",
expected: TestExpected{
WhereClause: "ifNull(dynamicElement(body_v2.`user.name`, 'String') = ?, false)",
Args: []any{"alice", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
{
name: "Scalar leaf hasAny",
filter: "hasAny(body.user.name, ['alice', 'bob'])",
expected: TestExpected{
WhereClause: "ifNull(dynamicElement(body_v2.`user.name`, 'String') IN (?, ?), false)",
Args: []any{"alice", "bob", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
{
// contains-all: single-value hasAll over a plain scalar collapses to has.
name: "Scalar leaf hasAll single collapses to has",
filter: "hasAll(body.user.name, 'alice')",
expected: TestExpected{
WhereClause: "ifNull(dynamicElement(body_v2.`user.name`, 'String') = ?, false)",
Args: []any{"alice", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
{
// contains-all over a one-element set: the scalar must equal every value, so
// distinct values never match.
name: "Scalar leaf hasAll multi (contains-all)",
filter: "hasAll(body.user.name, ['alice', 'bob'])",
expected: TestExpected{
WhereClause: "(ifNull(dynamicElement(body_v2.`user.name`, 'String') = ?, false) AND ifNull(dynamicElement(body_v2.`user.name`, 'String') = ?, false))",
Args: []any{"alice", "bob", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},
}
for _, c := range cases {

View File

@@ -130,3 +130,121 @@ func GetBodyJSONKey(_ context.Context, key *telemetrytypes.TelemetryFieldKey, op
func GetBodyJSONKeyForExists(_ context.Context, key *telemetrytypes.TelemetryFieldKey, _ qbtypes.FilterOperator, _ any) string {
return fmt.Sprintf("JSON_EXISTS(body, '$.%s')", getBodyJSONPath(key))
}
// legacyElemType infers the has-family element type from the needle (legacy has no schema). It
// scans EVERY value so the chosen array type and all coerced needles agree — else ClickHouse
// raises "no supertype ... String" (code 386). Int64 stays distinct from Float64 so a quoted
// integer is exact past 2^53 (unquoted literals already arrive as float64, parsed upstream).
func legacyElemType(needle any) telemetrytypes.FieldDataType {
list, ok := needle.([]any)
if !ok {
list = []any{needle}
}
if len(list) == 0 {
return telemetrytypes.FieldDataTypeString
}
allInt, allNumeric := true, true
for _, v := range list {
switch t := v.(type) {
case float32, float64:
allInt = false
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
// integer Go types stay int-exact
case string:
if _, err := strconv.ParseInt(t, 10, 64); err != nil {
allInt = false
}
if _, err := strconv.ParseFloat(t, 64); err != nil {
allNumeric = false
}
default:
// booleans (and anything else) -> String; a bool renders to 'true'/'false', so a
// bool needle only matches genuine JSON booleans, not truthy numbers/strings.
allInt, allNumeric = false, false
}
}
switch {
case allInt:
return telemetrytypes.FieldDataTypeInt64
case allNumeric:
return telemetrytypes.FieldDataTypeFloat64
default:
return telemetrytypes.FieldDataTypeString
}
}
// legacyCoerceNeedle coerces a needle to elem type dt so its bound-arg type matches the
// extracted column (legacyElemType guarantees it's coercible).
func legacyCoerceNeedle(v any, dt telemetrytypes.FieldDataType) any {
switch dt {
case telemetrytypes.FieldDataTypeInt64:
if s, ok := v.(string); ok {
if i, err := strconv.ParseInt(s, 10, 64); err == nil {
return i
}
}
return v
case telemetrytypes.FieldDataTypeFloat64:
if s, ok := v.(string); ok {
f, _ := strconv.ParseFloat(s, 64)
return f
}
return v
default:
return bodyArrayNeedleString(v)
}
}
// getBodyJSONArrayKey extracts the leaf as Array(Nullable(<dt>)) — Nullable so a value of a
// different JSON type maps to NULL instead of corrupting (e.g. a non-numeric string → 0).
func getBodyJSONArrayKey(key *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType) string {
arrKey := *key
if !strings.HasSuffix(arrKey.Name, "[*]") && !strings.HasSuffix(arrKey.Name, "[]") {
arrKey.Name += "[*]"
}
return fmt.Sprintf("JSONExtract(JSON_QUERY(body, '$.%s'), 'Array(Nullable(%s))')", getBodyJSONPath(&arrKey), dt.CHDataType())
}
// getBodyJSONScalarKey builds the single-element-set fallback for a scalar body value: the leaf
// extracted as a scalar of type dt, plus a guard restricting it to a genuinely scalar body. The
// guard is required because JSON_VALUE returns '' for an array/object/missing value, which would
// otherwise zero-value match (has(x,0) / has(x,false) / has(x,'') on any array). ok=false when
// the path still traverses an array ([*]/[]).
func getBodyJSONScalarKey(key *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType) (expr string, guard string, ok bool) {
name := strings.TrimSuffix(strings.TrimSuffix(key.Name, "[*]"), "[]")
if strings.Contains(name, "[") {
return "", "", false
}
scalarKey := *key
scalarKey.Name = name
path := getBodyJSONPath(&scalarKey)
if dt == telemetrytypes.FieldDataTypeString {
expr = fmt.Sprintf("JSON_VALUE(body, '$.%s')", path)
} else {
// Nullable so a scalar of a different type (e.g. a bool/string where a number is
// searched) extracts to NULL rather than the type's default (0/false), which would
// otherwise zero-value match has(x, 0).
expr = fmt.Sprintf("JSONExtract(JSON_VALUE(body, '$.%s'), 'Nullable(%s)')", path, dt.CHDataType())
}
keys := strings.Split(name, ".")
for i, k := range keys {
keys[i] = "'" + k + "'"
}
guard = fmt.Sprintf("JSONType(body, %s) NOT IN ('Array', 'Object', 'Null')", strings.Join(keys, ", "))
return expr, guard, true
}
func bodyArrayNeedleString(v any) string {
switch t := v.(type) {
case string:
return t
case bool:
return strconv.FormatBool(t)
case float64:
return strconv.FormatFloat(t, 'f', -1, 64)
case float32:
return strconv.FormatFloat(float64(t), 'f', -1, 64)
default:
return fmt.Sprintf("%v", t)
}
}

View File

@@ -495,8 +495,8 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
Limit: 10,
},
expected: qbtypes.Statement{
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE has(JSONExtract(JSON_QUERY(body, '$.\"user_names\"[*]'), 'Array(String)'), ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"john_doe", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (has(JSONExtract(JSON_QUERY(body, '$.\"user_names\"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$.\"user_names\"') = ? AND JSONType(body, 'user_names') NOT IN ('Array', 'Object', 'Null')), false)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"john_doe", "john_doe", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
expectedErr: nil,
},

View File

@@ -68,38 +68,6 @@ func NewTuplesFromTransactionGroups(name string, orgID valuer.UUID, transactionG
return tuples, nil
}
func DiffTuples(existing, desired []*openfgav1.TupleKey) (additions, deletions []*openfgav1.TupleKey) {
key := func(tuple *openfgav1.TupleKey) string {
return tuple.GetUser() + "|" + tuple.GetRelation() + "|" + tuple.GetObject()
}
existingSet := make(map[string]struct{}, len(existing))
for _, tuple := range existing {
existingSet[key(tuple)] = struct{}{}
}
desiredSet := make(map[string]struct{}, len(desired))
for _, tuple := range desired {
desiredSet[key(tuple)] = struct{}{}
}
additions = make([]*openfgav1.TupleKey, 0)
for _, tuple := range desired {
if _, ok := existingSet[key(tuple)]; !ok {
additions = append(additions, tuple)
}
}
deletions = make([]*openfgav1.TupleKey, 0)
for _, tuple := range existing {
if _, ok := desiredSet[key(tuple)]; !ok {
deletions = append(deletions, tuple)
}
}
return additions, deletions
}
func MustNewTransactionGroupsFromTuples(tuples []*openfgav1.TupleKey) TransactionGroups {
objectsByRelation := make(map[string][]*coretypes.Object)

View File

@@ -1,72 +0,0 @@
package authtypes
import (
"testing"
"github.com/SigNoz/signoz/pkg/types/coretypes"
"github.com/SigNoz/signoz/pkg/valuer"
openfgav1 "github.com/openfga/api/proto/openfga/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestTelemetryGrantAndCheckObjectsMatch(t *testing.T) {
orgID := valuer.GenerateUUID()
grantGroups := TransactionGroups{
{
Relation: Relation{Verb: coretypes.VerbRead},
ObjectGroup: coretypes.ObjectGroup{
Resource: coretypes.ResourceRef{Type: coretypes.TypeTelemetryResource, Kind: coretypes.KindLogs},
Selectors: []coretypes.Selector{coretypes.TypeTelemetryResource.MustSelector("builder_query/service.name/checkout")},
},
},
}
grantTuples, err := NewTuplesFromTransactionGroups("scoped-role", orgID, grantGroups)
require.NoError(t, err)
require.Len(t, grantTuples, 1)
checkTuples := NewTuples(
coretypes.ResourceTelemetryResourceLogs,
"user:organization/"+orgID.StringValue()+"/user/some-user",
Relation{Verb: coretypes.VerbRead},
[]coretypes.Selector{
coretypes.TypeTelemetryResource.MustSelector("builder_query/service.name/checkout"),
coretypes.TypeTelemetryResource.MustSelector("builder_query/service.name/payments"),
coretypes.TypeTelemetryResource.MustSelector("promql/service.name/checkout"),
coretypes.TypeTelemetryResource.MustSelector("builder_query/service.name/*"),
coretypes.TypeTelemetryResource.MustSelector("builder_query/*"),
coretypes.TypeTelemetryResource.MustSelector(coretypes.WildCardSelectorString),
},
orgID,
)
require.Len(t, checkTuples, 6)
assert.Equal(t, grantTuples[0].GetObject(), checkTuples[0].GetObject())
assert.NotEqual(t, grantTuples[0].GetObject(), checkTuples[1].GetObject())
assert.NotEqual(t, grantTuples[0].GetObject(), checkTuples[2].GetObject())
assert.NotContains(t, checkTuples[0].GetObject(), "checkout")
assert.Equal(t, "telemetryresource:organization/"+orgID.StringValue()+"/logs/builder_query/service.name/*", checkTuples[3].GetObject())
assert.Equal(t, "telemetryresource:organization/"+orgID.StringValue()+"/logs/builder_query/*", checkTuples[4].GetObject())
assert.Equal(t, "telemetryresource:organization/"+orgID.StringValue()+"/logs/*", checkTuples[5].GetObject())
}
func TestDiffTuples(t *testing.T) {
tuple := func(object string) *openfgav1.TupleKey {
return &openfgav1.TupleKey{User: "role:organization/o/role/r#assignee", Relation: "read", Object: object}
}
existing := []*openfgav1.TupleKey{tuple("a"), tuple("b")}
desired := []*openfgav1.TupleKey{tuple("b"), tuple("c")}
additions, deletions := DiffTuples(existing, desired)
require.Len(t, additions, 1)
assert.Equal(t, "c", additions[0].GetObject())
require.Len(t, deletions, 1)
assert.Equal(t, "a", deletions[0].GetObject())
additions, deletions = DiffTuples(existing, existing)
assert.Empty(t, additions)
assert.Empty(t, deletions)
}

View File

@@ -55,13 +55,6 @@ func OneID(extractor ResourceIDExtractor) ResourceIDsExtractor {
}}
}
type ResourceWithID struct {
Resource Resource
ID string
}
type ResourceExtractor func(ExtractorContext) ([]ResourceWithID, error)
func PathParam(name string) ResourceIDExtractor {
return ResourceIDExtractor{Phase: PhaseRequest, Fn: func(ec ExtractorContext) (string, error) {
if ec.Request == nil {

View File

@@ -66,7 +66,7 @@ func MustNewObjectFromString(input string) *Object {
return &Object{Resource: resource, Selector: typed.MustSelector(orgParts[1])}
}
parts := strings.SplitN(input, "/", 4)
parts := strings.Split(input, "/")
if len(parts) != 4 {
panic(errors.Newf(errors.TypeInternal, errors.CodeInternal, "invalid input format: %s", input))
}

View File

@@ -23,5 +23,5 @@ var (
TypeRole = Type{valuer.NewString("role"), regexp.MustCompile(`^([a-z-]{1,50}|\*)$`), []Verb{VerbAssignee, VerbCreate, VerbList, VerbRead, VerbUpdate, VerbDelete, VerbAttach, VerbDetach}}
TypeOrganization = Type{valuer.NewString("organization"), regexp.MustCompile(`^(^[0-9a-f]{8}(?:\-[0-9a-f]{4}){3}-[0-9a-f]{12}$|\*)$`), []Verb{VerbRead, VerbUpdate}}
TypeMetaResource = Type{valuer.NewString("metaresource"), regexp.MustCompile(`^(^[0-9a-f]{8}(?:\-[0-9a-f]{4}){3}-[0-9a-f]{12}$|\*)$`), []Verb{VerbCreate, VerbList, VerbRead, VerbUpdate, VerbDelete, VerbAttach, VerbDetach}}
TypeTelemetryResource = Type{valuer.NewString("telemetryresource"), regexp.MustCompile(`^(\*|\S(.{0,253}\S)?)$`), []Verb{VerbRead}}
TypeTelemetryResource = Type{valuer.NewString("telemetryresource"), regexp.MustCompile(`^\*$`), []Verb{VerbRead}}
)

View File

@@ -30,19 +30,6 @@ func NewResolvedResource(
return resolved
}
func NewResolvedResourceWithID(verb Verb, category ActionCategory, resource Resource, id string, selector SelectorFunc) ResolvedResource {
resolved := &resolvedResource{verb: verb, category: category, resource: resource, selector: selector}
if id != "" {
resolved.ids = []string{id}
}
return resolved
}
func NewResolvedResourceWithError(verb Verb, category ActionCategory, err error) ResolvedResource {
return &resolvedResource{verb: verb, category: category, err: err}
}
func (resolved *resolvedResource) fill(phase ExtractPhase, ec ExtractorContext) {
if !resolved.idExtractor.IsPhase(phase) {
return

View File

@@ -1,12 +1,6 @@
package coretypes
import (
"crypto/sha256"
"encoding/hex"
"maps"
"slices"
"strings"
"github.com/SigNoz/signoz/pkg/valuer"
)
@@ -31,64 +25,8 @@ func (resourceTelemetryResource *resourceTelemetryResource) Prefix(orgID valuer.
return resourceTelemetryResource.Type().StringValue() + ":" + "organization" + "/" + orgID.StringValue() + "/" + resourceTelemetryResource.Kind().String()
}
var telemetryQueryTypeSelectors = map[string]struct{}{
"builder_query": {},
"builder_sub_query": {},
"builder_trace_operator": {},
"promql": {},
"clickhouse_sql": {},
}
var telemetryGrantKeys = map[string]struct{}{
"service.name": {},
}
func IsTelemetryQueryTypeSelector(selector string) bool {
_, ok := telemetryQueryTypeSelectors[selector]
return ok
}
func IsTelemetryGrantKey(key string) bool {
_, ok := telemetryGrantKeys[key]
return ok
}
func TelemetryGrantKeys() []string {
return slices.Sorted(maps.Keys(telemetryGrantKeys))
}
func (resourceTelemetryResource *resourceTelemetryResource) Object(orgID valuer.UUID, selector string) string {
prefix := resourceTelemetryResource.Prefix(orgID)
if selector == WildCardSelectorString {
return prefix + "/" + selector
}
parts := strings.SplitN(selector, "/", 3)
if !IsTelemetryQueryTypeSelector(parts[0]) {
return prefix + "/" + TelemetrySelectorSegment(selector)
}
if len(parts) == 2 && parts[1] == WildCardSelectorString {
return prefix + "/" + selector
}
if len(parts) == 3 && IsTelemetryGrantKey(parts[1]) {
if parts[2] == WildCardSelectorString {
return prefix + "/" + selector
}
return prefix + "/" + parts[0] + "/" + parts[1] + "/" + TelemetrySelectorSegment(parts[2])
}
return prefix + "/" + TelemetrySelectorSegment(selector)
}
// Must stay stable: grant-time and check-time object building both rely on
// producing the same segment for the same selector value.
func TelemetrySelectorSegment(selector string) string {
sum := sha256.Sum256([]byte(selector))
return hex.EncodeToString(sum[:16])
return resourceTelemetryResource.Prefix(orgID) + "/" + selector
}
func (resourceTelemetryResource *resourceTelemetryResource) Scope(verb Verb) string {

View File

@@ -1,62 +0,0 @@
package coretypes
import (
"strings"
"testing"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/stretchr/testify/assert"
)
func TestTelemetryResourceSelectorRegex(t *testing.T) {
valid := []string{
"*",
"a",
"checkout-service",
"signoz agent",
"frontend/us-east-1",
"abcdef0123456789abcdef0123456789",
strings.Repeat("a", 255),
}
for _, value := range valid {
_, err := TypeTelemetryResource.Selector(value)
assert.NoError(t, err, "expected %q to be a valid telemetry selector", value)
}
invalid := []string{
"",
" ",
" leading-space",
"trailing-space ",
strings.Repeat("a", 256),
}
for _, value := range invalid {
_, err := TypeTelemetryResource.Selector(value)
assert.Error(t, err, "expected %q to be rejected as a telemetry selector", value)
}
}
func TestTelemetrySelectorSegment(t *testing.T) {
segment := TelemetrySelectorSegment("checkout-service")
assert.Len(t, segment, 32)
assert.Equal(t, segment, TelemetrySelectorSegment("checkout-service"))
assert.NotEqual(t, segment, TelemetrySelectorSegment("checkout-service2"))
}
func TestTelemetryResourceObjectSelectors(t *testing.T) {
orgID := valuer.GenerateUUID()
prefix := "telemetryresource:organization/" + orgID.StringValue() + "/logs/"
assert.Equal(t, prefix+"*", ResourceTelemetryResourceLogs.Object(orgID, "*"))
assert.Equal(t, prefix+"promql/*", ResourceTelemetryResourceLogs.Object(orgID, "promql/*"))
assert.Equal(t, prefix+"builder_query/*", ResourceTelemetryResourceLogs.Object(orgID, "builder_query/*"))
assert.Equal(t, prefix+"builder_query/service.name/*", ResourceTelemetryResourceLogs.Object(orgID, "builder_query/service.name/*"))
assert.Equal(t, prefix+"builder_query/service.name/"+TelemetrySelectorSegment("checkout"), ResourceTelemetryResourceLogs.Object(orgID, "builder_query/service.name/checkout"))
assert.Equal(t, prefix+"builder_query/service.name/"+TelemetrySelectorSegment("a/b"), ResourceTelemetryResourceLogs.Object(orgID, "builder_query/service.name/a/b"))
assert.Equal(t, prefix+TelemetrySelectorSegment("builder_query/unknown.key/checkout"), ResourceTelemetryResourceLogs.Object(orgID, "builder_query/unknown.key/checkout"))
assert.Equal(t, prefix+TelemetrySelectorSegment("service.name = 'checkout'"), ResourceTelemetryResourceLogs.Object(orgID, "service.name = 'checkout'"))
object := MustNewObjectFromString(prefix + "builder_query/service.name/" + TelemetrySelectorSegment("checkout"))
assert.Equal(t, "builder_query/service.name/"+TelemetrySelectorSegment("checkout"), object.Selector.String())
assert.Equal(t, KindLogs, object.Resource.Kind)
}

View File

@@ -563,214 +563,6 @@ def test_logs_json_body_nested_keys(
assert all(code == 200 for code in status_codes)
def test_logs_json_body_array_membership(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
Insert logs with JSON bodies containing arrays
Tests:
1. Search by has(body.tags[*], "value") - string array
2. Search by has(body.ids[*], 123) - numeric array
3. Search by has(body.flags[*], true) - boolean array
"""
now = datetime.now(tz=UTC)
log1_body = json.dumps(
{
"tags": ["production", "api", "critical"],
"ids": [100, 200, 300],
"flags": [True, False, True],
"users": [
{"name": "alice", "role": "admin"},
{"name": "bob", "role": "user"},
],
}
)
log2_body = json.dumps(
{
"tags": ["staging", "api", "test"],
"ids": [200, 400, 500],
"flags": [False, False, True],
"users": [
{"name": "charlie", "role": "user"},
{"name": "david", "role": "admin"},
],
}
)
log3_body = json.dumps(
{
"tags": ["production", "web", "important"],
"ids": [100, 600, 700],
"flags": [True, True, False],
"users": [
{"name": "alice", "role": "admin"},
{"name": "eve", "role": "user"},
],
}
)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=3),
resources={"service.name": "app-service"},
attributes={},
body=log1_body,
severity_text="INFO",
),
Logs(
timestamp=now - timedelta(seconds=2),
resources={"service.name": "app-service"},
attributes={},
body=log2_body,
severity_text="INFO",
),
Logs(
timestamp=now - timedelta(seconds=1),
resources={"service.name": "app-service"},
attributes={},
body=log3_body,
severity_text="INFO",
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Test 1: Search by has(body.tags[*], "production")
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
timeout=2,
headers={"authorization": f"Bearer {token}"},
json={
"schemaVersion": "v1",
"start": int((now - timedelta(seconds=10)).timestamp() * 1000),
"end": int(now.timestamp() * 1000),
"requestType": "raw",
"compositeQuery": {
"queries": [
{
"type": "builder_query",
"spec": {
"name": "A",
"signal": "logs",
"disabled": False,
"limit": 100,
"offset": 0,
"filter": {"expression": 'has(body.tags[*], "production")'},
"order": [
{"key": {"name": "timestamp"}, "direction": "desc"},
],
"aggregations": [{"expression": "count()"}],
},
}
]
},
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
},
)
assert response.status_code == HTTPStatus.OK
assert response.json()["status"] == "success"
results = response.json()["data"]["data"]["results"]
assert len(results) == 1
rows = results[0]["rows"]
assert len(rows) == 2 # log1 and log3 have "production" in tags
tags_list = [json.loads(row["data"]["body"])["tags"] for row in rows]
assert all("production" in tags for tags in tags_list)
# Test 2: Search by has(body.ids[*], 200)
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
timeout=2,
headers={"authorization": f"Bearer {token}"},
json={
"schemaVersion": "v1",
"start": int((now - timedelta(seconds=10)).timestamp() * 1000),
"end": int(now.timestamp() * 1000),
"requestType": "raw",
"compositeQuery": {
"queries": [
{
"type": "builder_query",
"spec": {
"name": "A",
"signal": "logs",
"disabled": False,
"limit": 100,
"offset": 0,
"filter": {"expression": "has(body.ids[*], 200)"},
"order": [
{"key": {"name": "timestamp"}, "direction": "desc"},
],
"aggregations": [{"expression": "count()"}],
},
}
]
},
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
},
)
assert response.status_code == HTTPStatus.OK
assert response.json()["status"] == "success"
results = response.json()["data"]["data"]["results"]
assert len(results) == 1
rows = results[0]["rows"]
assert len(rows) == 2 # log1 and log2 have 200 in ids
ids_list = [json.loads(row["data"]["body"])["ids"] for row in rows]
assert all(200 in ids for ids in ids_list)
# Test 3: Search by has(body.flags[*], true)
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
timeout=2,
headers={"authorization": f"Bearer {token}"},
json={
"schemaVersion": "v1",
"start": int((now - timedelta(seconds=10)).timestamp() * 1000),
"end": int(now.timestamp() * 1000),
"requestType": "raw",
"compositeQuery": {
"queries": [
{
"type": "builder_query",
"spec": {
"name": "A",
"signal": "logs",
"disabled": False,
"limit": 100,
"offset": 0,
"filter": {"expression": "has(body.flags[*], true)"},
"order": [
{"key": {"name": "timestamp"}, "direction": "desc"},
],
"aggregations": [{"expression": "count()"}],
},
}
]
},
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
},
)
assert response.status_code == HTTPStatus.OK
assert response.json()["status"] == "success"
results = response.json()["data"]["data"]["results"]
assert len(results) == 1
rows = results[0]["rows"]
assert len(rows) == 3 # All logs have true in flags
flags_list = [json.loads(row["data"]["body"])["flags"] for row in rows]
assert all(True in flags for flags in flags_list)
def test_logs_json_body_listing(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument

File diff suppressed because it is too large Load Diff