Compare commits

..

30 Commits

Author SHA1 Message Date
Nikhil Soni
ab3b88966e Merge remote-tracking branch 'origin/main' into ns/scope
# Conflicts:
#	pkg/telemetryschema/tracestelemetryschema/field_mapper.go
#	tests/conftest.py
#	tests/integration/tests/querier/04_traces.py
2026-08-05 16:44:41 +05:30
Nikhil Soni
08ebc37109 fix: handle fixed scope.name and scope.version fields to work without scope 2026-07-02 19:04:41 +05:30
Nikhil Soni
aa5a1c5e62 chore: update collector version 2026-07-02 14:46:07 +05:30
Nikhil Soni
7b34a47ac5 chore: run formatting on tests 2026-07-02 14:41:33 +05:30
Nikhil Soni
b4b2d7bb66 test: add test to show cross context matching 2026-06-24 18:34:38 +05:30
Nikhil Soni
e16416475b refactor: drop unused fields 2026-06-24 18:07:47 +05:30
Nikhil Soni
0ea7c1ae6e test: add more cases for scope name 2026-06-24 18:05:50 +05:30
Nikhil Soni
a023c8ed4a test: add integration test for scope fields 2026-06-24 15:25:17 +05:30
Nikhil Soni
a73ae62cd1 Merge remote-tracking branch 'origin/main' into ns/scope 2026-06-24 13:02:18 +05:30
Nikhil Soni
ec6fb58052 chore: add more tests 2026-06-24 12:37:42 +05:30
Nikhil Soni
d3d13eb7ff fix: remove handling of normalized properties for scope
Otherwise it will be impossible to query if scope attribute also
exists with same name - name and version
2026-05-21 11:19:22 +05:30
Nikhil Soni
782de2b210 fix: use correct error type for internal issues 2026-05-20 15:32:26 +05:30
Nikhil Soni
d3c38693f3 fix: allow 'scope.' prefix for keys with other context 2026-05-20 15:30:25 +05:30
Nikhil Soni
8791df3697 fix: avoid removing context prefix to support attr with prefix 2026-05-19 19:14:23 +05:30
Nikhil Soni
eb719c3d0d fix: use key selector with context prefix 2026-05-19 18:56:00 +05:30
Nikhil Soni
f10435c210 Merge remote-tracking branch 'origin' into ns/scope 2026-05-19 13:55:36 +05:30
Nikhil Soni
f3f1e9cb59 chore: add tests for denormalized field name as well 2026-05-19 13:55:25 +05:30
Nikhil Soni
d0370ce3ef fix: handle fields with included context for scope (select clause) 2026-05-14 17:02:56 +05:30
Nikhil Soni
d169761e65 Merge remote-tracking branch 'origin/main' into ns/scope 2026-05-14 11:50:33 +05:30
Nikhil Soni
87864ef5d4 chore: remove duplicates from .gitignore 2026-05-11 15:45:32 +05:30
Nikhil Soni
2e0bc8998e chore: use name as key name for scope instead of scope.name 2026-05-11 15:40:45 +05:30
Nikhil Soni
7e1f4aa50d Merge remote-tracking branch 'origin/main' into ns/scope 2026-05-11 14:27:19 +05:30
Nikhil Soni
35da39247c Merge branch 'main' into ns/scope 2026-05-07 17:41:11 +05:30
Nikhil Soni
ceccc47a34 fix: fix test for case without resource filter 2026-05-07 16:04:03 +05:30
Nikhil Soni
23da5e22ec Merge branch 'main' into ns/scope 2026-05-07 13:27:34 +05:30
Nikhil Soni
4c1b479149 chore: add tests for scope fields 2026-04-28 20:27:10 +05:30
Nikhil Soni
f72204a8b2 refactor: simplify field mapper for scope 2026-04-28 20:26:37 +05:30
Nikhil Soni
deb3f385fa chore: remove underscore version of scope fields 2026-04-23 10:26:55 +05:30
Nikhil Soni
77ce5f86b1 fix: use scope as json field instead with name and version 2026-04-23 01:15:02 +05:30
Nikhil Soni
ff211de441 feat: add support for scope fields in traces 2026-04-14 10:45:08 +05:30
28 changed files with 519 additions and 633 deletions

3
.gitignore vendored
View File

@@ -231,4 +231,5 @@ cython_debug/
# LSP config files
pyrightconfig.json
# agents
.claude/settings.local.json

View File

@@ -182,56 +182,4 @@ describe('ValueSelector', () => {
});
});
});
describe('opening and closing without touching the list', () => {
function renderWith(
selection: VariableSelection,
options: string[],
): jest.Mock {
const onChange = jest.fn();
render(
<TooltipProvider>
<ValueSelector
options={options}
variableType="dynamic"
multiSelect
showAllOption
selection={selection}
onChange={onChange}
emptyFallback={{ value: [], allSelected: false }}
testId="variable-select-env"
/>
</TooltipProvider>,
);
return onChange;
}
async function openThenClose(): Promise<void> {
const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime });
const control = screen.getByTestId('variable-select-env');
await user.click(control.querySelector('input') as HTMLInputElement);
await user.keyboard('{Escape}');
}
it('does not promote a pick that covers every available option to ALL', async () => {
// A narrow time range can leave only the selected value in the list. That is
// still an explicit pick, not "everything, always".
const onChange = renderWith(
{ value: ['checkout-service-prod'], allSelected: false },
['checkout-service-prod'],
);
await openThenClose();
expect(onChange).not.toHaveBeenCalled();
});
it('does not rewrite a dynamic ALL into concrete values', async () => {
const onChange = renderWith({ value: null, allSelected: true }, OPTIONS);
await openThenClose();
expect(onChange).not.toHaveBeenCalled();
});
});
});

View File

@@ -145,133 +145,6 @@ describe('reconcileWithOptions', () => {
),
).toBeNull();
});
describe('preserveSelection (options moved on their own — time range, reload)', () => {
const multi = model({
type: 'DYNAMIC',
multiSelect: true,
showAllOption: true,
dynamicAttribute: 'service.name',
});
it('keeps a multi-select pick the new option list no longer offers', () => {
expect(
reconcileWithOptions(multi, { value: ['frontend'], allSelected: false }, [
'backend',
'cart',
]),
).toStrictEqual({ value: null, allSelected: true });
expect(
reconcileWithOptions(
multi,
{ value: ['frontend'], allSelected: false },
['backend', 'cart'],
{ preserveSelection: true },
),
).toBeNull();
});
it('still materializes ALL, which must track the option list', () => {
expect(
reconcileWithOptions(
model({ type: 'QUERY', multiSelect: true, showAllOption: true }),
{ value: ['a'], allSelected: true },
['a', 'b'],
{ preserveSelection: true },
),
).toStrictEqual({ value: ['a', 'b'], allSelected: true });
});
it('still fills the default when nothing is selected yet', () => {
expect(
reconcileWithOptions(multi, { value: [], allSelected: false }, ['a', 'b'], {
preserveSelection: true,
}),
).toStrictEqual({ value: null, allSelected: true });
});
});
// A typed value is in no option list, so no refetch can invalidate it.
describe('customValues (typed in, never offered by the data)', () => {
const multi = model({
type: 'DYNAMIC',
multiSelect: true,
showAllOption: true,
dynamicAttribute: 'service.name',
});
it('keeps them through a re-scope that drops a fetched value', () => {
expect(
reconcileWithOptions(
multi,
{
value: ['frontend', 'typed-in'],
allSelected: false,
customValues: ['typed-in'],
},
['backend', 'cart'],
),
).toStrictEqual({
value: ['typed-in'],
allSelected: false,
customValues: ['typed-in'],
});
});
it('never re-defaults a selection made only of them', () => {
expect(
reconcileWithOptions(
multi,
{ value: ['typed-in'], allSelected: false, customValues: ['typed-in'] },
['backend', 'cart'],
),
).toBeNull();
});
// An inert marker is not worth a store write + dependent refetch to prune.
it('leaves a stale marker alone when it drops nothing', () => {
expect(
reconcileWithOptions(
multi,
{
value: ['frontend', 'typed-in'],
allSelected: false,
customValues: ['typed-in', 'removed-earlier'],
},
['frontend'],
),
).toBeNull();
});
it('prunes markers for values it does drop', () => {
expect(
reconcileWithOptions(
multi,
{
value: ['stale', 'typed-in'],
allSelected: false,
customValues: ['typed-in'],
},
['frontend'],
),
).toStrictEqual({
value: ['typed-in'],
allSelected: false,
customValues: ['typed-in'],
});
});
it('still drops an unmarked value the list no longer offers', () => {
expect(
reconcileWithOptions(
multi,
{ value: ['frontend', 'stale'], allSelected: false },
['frontend'],
),
).toStrictEqual({ value: ['frontend'], allSelected: false });
});
});
});
describe('configuredDefaultValue', () => {

View File

@@ -1,91 +0,0 @@
import type { VariableSelection } from '../selectionTypes';
import { selectionFromCommittedValues } from '../utils/selectionUtils';
const OPTIONS = ['checkout', 'payments', 'cart'];
const FALLBACK: VariableSelection = { value: null, allSelected: true };
function commit(
values: string[],
overrides: Partial<Parameters<typeof selectionFromCommittedValues>[0]> = {},
): VariableSelection {
return selectionFromCommittedValues({
values,
options: OPTIONS,
showAllOption: true,
emptyFallback: FALLBACK,
...overrides,
});
}
// What a multi-select commit resolves to. The option list is known only here, so this
// is the one place a typed value can be recognised.
describe('selectionFromCommittedValues', () => {
it('marks values the option list did not offer as typed in', () => {
expect(commit(['checkout', 'typed-in'])).toStrictEqual({
value: ['checkout', 'typed-in'],
allSelected: false,
customValues: ['typed-in'],
});
});
it('marks a selection made only of typed-in values', () => {
expect(commit(['a', 'b'])).toStrictEqual({
value: ['a', 'b'],
allSelected: false,
customValues: ['a', 'b'],
});
});
it('records no marker when every pick came from the list', () => {
expect(commit(['checkout', 'cart'])).toStrictEqual({
value: ['checkout', 'cart'],
allSelected: false,
});
});
it('reads a set covering every option as ALL', () => {
expect(commit(OPTIONS)).toStrictEqual({
value: OPTIONS,
allSelected: true,
});
});
// ALL re-materializes to the option set, so recording this as ALL would drop the
// typed value on the next refetch.
it('does not read every option PLUS a typed value as ALL', () => {
expect(commit([...OPTIONS, 'typed-in'])).toStrictEqual({
value: [...OPTIONS, 'typed-in'],
allSelected: false,
customValues: ['typed-in'],
});
});
// Derived from the values + options at commit time, never from the old selection.
it('recomputes the marker: a typed value the data now offers is a normal pick', () => {
expect(
commit(['checkout', 'was-typed'], {
options: [...OPTIONS, 'was-typed'],
}),
).toStrictEqual({ value: ['checkout', 'was-typed'], allSelected: false });
});
it('does not read it as ALL when the variable offers no ALL', () => {
expect(commit(OPTIONS, { showAllOption: false })).toStrictEqual({
value: OPTIONS,
allSelected: false,
});
});
it('resolves an empty commit to the variable fallback', () => {
expect(commit([])).toBe(FALLBACK);
});
it('marks everything while the options have not arrived', () => {
// Nothing to judge against yet; erring this way keeps a value rather than dropping it.
expect(commit(['typed-in'], { options: [] })).toStrictEqual({
value: ['typed-in'],
allSelected: false,
customValues: ['typed-in'],
});
});
});

View File

@@ -4,8 +4,6 @@ import {
emptyVariableFormModel,
type VariableFormModel,
} from '../../DashboardSettings/Variables/variableFormModel';
import { VariableCycleReason } from '../../store/slices/variableFetchSlice';
import { useDashboardStore } from '../../store/useDashboardStore';
import type { VariableSelection } from '../selectionTypes';
import { useAutoSelect } from '../hooks/useAutoSelect';
@@ -17,11 +15,7 @@ function run(
variable: VariableFormModel,
options: string[],
selection: VariableSelection,
cycleReason?: VariableCycleReason,
): VariableSelection | undefined {
useDashboardStore.setState({
variableCycleReasons: cycleReason ? { [variable.name]: cycleReason } : {},
});
const onAutoSelect = jest.fn();
renderHook(() => useAutoSelect(variable, options, selection, onAutoSelect));
return onAutoSelect.mock.calls[0]?.[0];
@@ -76,13 +70,11 @@ describe('useAutoSelect', () => {
expect(next).toStrictEqual({ value: ['a', 'b'], allSelected: true });
});
// Re-scoped options only — a time-range refetch must NOT re-default; see below.
it('re-scoped: falls back to ALL, not the first option, when every selected value is gone', () => {
it('falls back to ALL, not the first option, when every selected value is gone', () => {
const next = run(
model({ type: 'QUERY', multiSelect: true, showAllOption: true }),
['x', 'y'],
{ value: ['a', 'b'], allSelected: false },
VariableCycleReason.ValueCascade,
);
expect(next).toStrictEqual({ value: ['x', 'y'], allSelected: true });
});
@@ -110,23 +102,20 @@ describe('useAutoSelect', () => {
expect(next).toStrictEqual({ value: ['b'], allSelected: false });
});
it('re-scoped: keeps the still-valid subset of a multi-select', () => {
it('keeps the still-valid subset of a multi-select when options re-scope', () => {
const next = run(
model({ type: 'QUERY', multiSelect: true }),
['a', 'b', 'd'],
{ value: ['a', 'b', 'c'], allSelected: false },
VariableCycleReason.ValueCascade,
);
expect(next).toStrictEqual({ value: ['a', 'b'], allSelected: false });
});
it('re-scoped: re-defaults a multi-select when none of the selected values remain', () => {
const next = run(
model({ type: 'QUERY', multiSelect: true }),
['x', 'y'],
{ value: ['a', 'b'], allSelected: false },
VariableCycleReason.ValueCascade,
);
it('re-defaults a multi-select when none of the selected values remain', () => {
const next = run(model({ type: 'QUERY', multiSelect: true }), ['x', 'y'], {
value: ['a', 'b'],
allSelected: false,
});
expect(next).toStrictEqual({ value: ['x'], allSelected: false });
});
@@ -162,45 +151,4 @@ describe('useAutoSelect', () => {
});
expect(next).toBeUndefined();
});
describe('by cycle reason', () => {
const service = model({
name: 'service',
type: 'DYNAMIC',
multiSelect: true,
showAllOption: true,
dynamicAttribute: 'service.name',
});
const gone: VariableSelection = { value: ['frontend'], allSelected: false };
it('keeps the selection when a full cycle refetched the options', () => {
// The new window has no data for the selected service — no reason to widen to ALL.
const next = run(
service,
['backend', 'cart'],
gone,
VariableCycleReason.FullCycle,
);
expect(next).toBeUndefined();
});
it('re-scopes the selection when a value cascade refetched the options', () => {
const next = run(
service,
['backend', 'cart'],
gone,
VariableCycleReason.ValueCascade,
);
expect(next).toStrictEqual({ value: null, allSelected: true });
});
it('reconciles a variable with no cycle of its own (custom definition change)', () => {
const next = run(
model({ name: 'env', type: 'CUSTOM', multiSelect: true }),
['staging', 'prod'],
{ value: ['dev'], allSelected: false },
);
expect(next).toStrictEqual({ value: ['staging'], allSelected: false });
});
});
});

View File

@@ -13,11 +13,11 @@ jest.mock('nuqs', () => ({
useQueryState: (): unknown => [null, jest.fn()],
}));
const mockGlobalTime = { minTime: 1, maxTime: 2, selectedTime: '5m' };
jest.mock('react-redux', () => ({
useSelector: (selector: (state: unknown) => unknown): unknown =>
selector({ globalTime: mockGlobalTime }),
selector({
globalTime: { minTime: 1, maxTime: 2, selectedTime: '5m' },
}),
}));
jest.mock('../../DashboardSettings/Variables/variableAdapters', () => ({
@@ -150,57 +150,3 @@ describe('useVariableSelection — setSelection', () => {
expect(svcCycleId()).toBe(before + 1);
});
});
describe('useVariableSelection — what a time-range change enqueues', () => {
// Longer than FETCH_CYCLE_DEBOUNCE_MS, which the hook keeps private.
const PAST_DEBOUNCE = 400;
function reasons(): Record<string, string> {
return useDashboardStore.getState().variableCycleReasons;
}
beforeEach(() => {
jest.useFakeTimers();
mockGlobalTime.selectedTime = '5m';
useDashboardStore.setState({
variableValues: {},
variableFetchStates: {},
variableLastUpdated: {},
variableCycleIds: {},
variableCycleReasons: {},
variableResolvedEmpty: {},
variableFetchContext: null,
lastFetchAllKey: null,
});
});
afterEach(() => {
jest.useRealTimers();
});
// The tag is what stops the reconcile re-defaulting a user's selection.
it('tags every variable as a full cycle, overriding an earlier cascade tag', () => {
const { result, rerender } = renderHook(() =>
useVariableSelection(dashboard),
);
act(() => {
jest.advanceTimersByTime(PAST_DEBOUNCE);
});
expect(reasons()).toStrictEqual({ env: 'full-cycle', svc: 'full-cycle' });
// A value change re-scopes the dependent's options: it may drop what no longer applies.
act(() => {
result.current.setSelection('env', { value: ['prod'], allSelected: false });
});
expect(reasons().svc).toBe('value-cascade');
mockGlobalTime.selectedTime = '30m';
rerender();
act(() => {
jest.advanceTimersByTime(PAST_DEBOUNCE);
});
expect(reasons()).toStrictEqual({ env: 'full-cycle', svc: 'full-cycle' });
});
});

View File

@@ -6,7 +6,6 @@ import { DashboardDetailEvents } from 'pages/DashboardPageV2/constants/events';
import type { VariableSelection } from '../../selectionTypes';
import { areSelectionsEqual } from '../../utils/resolveVariableSelection';
import { selectionFromCommittedValues } from '../../utils/selectionUtils';
import OverflowValuesTooltip from './OverflowValuesTooltip';
import styles from '../../VariablesBar.module.scss';
@@ -76,23 +75,13 @@ function ValueSelector({
options.every((option) => draft.includes(option));
const commit = (values: string[]): void => {
// A close that left the list as it opened commits nothing — else a pick covering
// every option this window offers would be promoted to a standing ALL.
if (
areSelectionsEqual(
{ value: values, allSelected: false },
{ value: committedValues, allSelected: false },
)
) {
return;
}
const next = selectionFromCommittedValues({
values,
options,
showAllOption,
emptyFallback,
});
// CustomMultiSelect emits the full value set when ALL is picked.
const isAll =
showAllOption &&
options.length > 0 &&
options.every((option) => values.includes(option));
const next: VariableSelection =
values.length === 0 ? emptyFallback : { value: values, allSelected: isAll };
// Closing without actually changing the selection must not re-fire onChange —
// that would needlessly re-cascade to dependent variables/panels.

View File

@@ -1,11 +1,6 @@
import { useEffect } from 'react';
import type { VariableFormModel } from '../../DashboardSettings/Variables/variableFormModel';
import {
selectVariableCycleReason,
VariableCycleReason,
} from '../../store/slices/variableFetchSlice';
import { useDashboardStore } from '../../store/useDashboardStore';
import { reconcileWithOptions } from '../utils/resolveVariableSelection';
import type { VariableSelection } from '../selectionTypes';
@@ -14,9 +9,6 @@ import type { VariableSelection } from '../selectionTypes';
* `onAutoSelect` only when the value must change. The reconcile rule lives in
* {@link reconcileWithOptions} (shared with seed + payload defaulting) so the bar
* and the panel query can never disagree about a variable's default.
*
* Only a value cascade may re-default the selection; a full cycle (time range,
* reload) leaves the user's pick alone. Types with no cycle of their own reconcile.
*/
export function useAutoSelect(
variable: VariableFormModel,
@@ -24,14 +16,8 @@ export function useAutoSelect(
selection: VariableSelection,
onAutoSelect: (selection: VariableSelection) => void,
): void {
const cycleReason = useDashboardStore(
selectVariableCycleReason(variable.name),
);
useEffect(() => {
const next = reconcileWithOptions(variable, selection, options, {
preserveSelection: cycleReason === VariableCycleReason.FullCycle,
});
const next = reconcileWithOptions(variable, selection, options);
if (next) {
onAutoSelect(next);
}

View File

@@ -10,11 +10,6 @@ export interface VariableSelection {
value: SelectedVariableValue;
/** True when every option is selected ("ALL"); for dynamic vars value may be null. */
allSelected: boolean;
/**
* Entries of `value` the user typed rather than picked. Never in any option list,
* so the reconcile keeps them instead of reading them as invalid.
*/
customValues?: string[];
}
/** Selected values for a dashboard's variables, keyed by variable name. */

View File

@@ -134,23 +134,12 @@ export function resolveDefaultSelection(
return { value: model.multiSelect ? [] : '', allSelected: false };
}
interface ReconcileOptions {
/**
* Set when no other variable caused this refetch (time-range change, reload): the
* selection then outranks the options and is kept as-is. Leave false for a
* dependency cascade, where a selection that no longer applies must give way.
*/
preserveSelection?: boolean;
}
/**
* Reconciles a variable's current selection against its freshly-fetched options.
* Returns the next selection, or null when nothing should change (a valid pick is
* left untouched — local-first). Behaviour, in order:
* - materialize ALL to the full option set (query/custom);
* - keep a multi-select selection outright when `preserveSelection` is set;
* - keep a still-valid multi-select subset, dropping only entries the list no longer
* offers and the user did not type in (`customValues`);
* - keep a still-valid multi-select subset, dropping only invalid entries;
* - otherwise auto-pick the default (or first option) so dependent variables and
* panels always resolve against a usable value.
*/
@@ -158,7 +147,6 @@ export function reconcileWithOptions(
model: VariableFormModel,
current: VariableSelection,
options: string[],
{ preserveSelection = false }: ReconcileOptions = {},
): VariableSelection | null {
if (options.length === 0) {
return null;
@@ -173,31 +161,13 @@ export function reconcileWithOptions(
Array.isArray(current.value) &&
current.value.length > 0
) {
// A pick this window has no data for is still the user's filter; re-defaulting it
// here is what widened a single pick to ALL on every time-range change.
if (preserveSelection) {
return null;
}
// A typed value is in no option list, so it is never "no longer offered".
const custom = new Set(current.customValues ?? []);
const valid = current.value
.map(String)
.filter((c) => options.includes(c) || custom.has(c));
const valid = current.value.map(String).filter((c) => options.includes(c));
if (valid.length === current.value.length) {
return null;
}
if (valid.length === 0) {
return fillDefault(model, options);
}
const customValues = valid.filter((v) => custom.has(v));
return {
value: valid,
allSelected: false,
...(customValues.length > 0 && { customValues }),
};
return valid.length > 0
? { value: valid, allSelected: false }
: fillDefault(model, options);
}
if (!model.multiSelect) {

View File

@@ -47,43 +47,6 @@ export function hasUsableValue(
return value !== '' && value !== null && value !== undefined;
}
interface CommittedValues {
values: string[];
options: string[];
showAllOption: boolean;
emptyFallback: VariableSelection;
}
/**
* The selection a multi-select commit resolves to. Options are known only here, so
* this is where a value the list never offered is recorded as typed in.
*/
export function selectionFromCommittedValues({
values,
options,
showAllOption,
emptyFallback,
}: CommittedValues): VariableSelection {
if (values.length === 0) {
return emptyFallback;
}
const customValues = values.filter((value) => !options.includes(value));
// ALL re-materializes to the option set, so a set carrying a typed value is not ALL
// — the next refetch would expand it back and drop what the user typed.
const allSelected =
showAllOption &&
options.length > 0 &&
customValues.length === 0 &&
options.every((option) => values.includes(option));
return {
value: values,
allSelected,
...(customValues.length > 0 && { customValues }),
};
}
/** Flatten the selection map into the `{ name: value }` payload a query expects. */
export function selectionToPayload(
selection: VariableSelectionMap,

View File

@@ -34,7 +34,6 @@ function reset(names: string[], context: VariableFetchContext): void {
variableFetchStates: {},
variableLastUpdated: {},
variableCycleIds: {},
variableCycleReasons: {},
variableFetchContext: null,
});
store().initVariableFetch(names, context);
@@ -134,33 +133,6 @@ describe('variableFetchSlice', () => {
expect(states().q1).toBe('error');
expect(states().q2).toBe('idle');
});
// The reason is what tells the post-fetch reconcile whether it may re-default a
// selection: a full cycle must not, a value cascade must.
it('tags a full cycle, then re-tags only the cascaded variables', () => {
store().enqueueFetchAll();
expect(store().variableCycleReasons).toStrictEqual({
q1: 'full-cycle',
q2: 'full-cycle',
d1: 'full-cycle',
d2: 'full-cycle',
});
resolve('q1');
store().enqueueDescendants('q1');
expect(store().variableCycleReasons).toStrictEqual({
q1: 'full-cycle',
q2: 'value-cascade',
d1: 'full-cycle',
d2: 'full-cycle',
});
});
it('drops the reason for a variable that no longer exists', () => {
store().enqueueFetchAll();
store().initVariableFetch(['q1'], context);
expect(store().variableCycleReasons).toStrictEqual({ q1: 'full-cycle' });
});
});
describe('variableFetchSlice — query depends on a dynamic', () => {

View File

@@ -9,7 +9,6 @@ import {
type FetchMaps,
isVariableInActiveFetchState,
resolveFetchState,
VariableCycleReason,
VariableFetchState,
} from './variableFetchSlice.utils';
@@ -31,10 +30,7 @@ function queryParentsHaveValues(
);
}
export {
VariableCycleReason,
VariableFetchState,
} from './variableFetchSlice.utils';
export { VariableFetchState } from './variableFetchSlice.utils';
/**
* Runtime fetch orchestration for dashboard variables — native port of V1's
@@ -49,8 +45,6 @@ export interface VariableFetchSlice {
variableFetchStates: Record<string, VariableFetchState>;
variableLastUpdated: Record<string, number>;
variableCycleIds: Record<string, number>;
/** Why each variable's current cycle was enqueued, read by the post-fetch reconcile. */
variableCycleReasons: Record<string, VariableCycleReason>;
/**
* Whether a QUERY/DYNAMIC variable settled its fetch with zero options (so it
* will never get a value). Lets a dependent panel fall through to "no data"
@@ -112,7 +106,6 @@ export const createVariableFetchSlice: StateCreator<
variableFetchStates: {},
variableLastUpdated: {},
variableCycleIds: {},
variableCycleReasons: {},
variableResolvedEmpty: {},
variableFetchContext: null,
lastFetchAllKey: null,
@@ -122,7 +115,6 @@ export const createVariableFetchSlice: StateCreator<
variableFetchStates: {},
variableLastUpdated: {},
variableCycleIds: {},
variableCycleReasons: {},
variableResolvedEmpty: {},
variableFetchContext: null,
lastFetchAllKey: null,
@@ -140,7 +132,6 @@ export const createVariableFetchSlice: StateCreator<
initVariableFetch: (names, context): void => {
const maps = cloneMaps(get());
const resolvedEmpty = { ...get().variableResolvedEmpty };
const reasons = { ...get().variableCycleReasons };
names.forEach((name) => {
if (!maps.states[name]) {
maps.states[name] = VariableFetchState.Idle;
@@ -153,14 +144,12 @@ export const createVariableFetchSlice: StateCreator<
delete maps.lastUpdated[name];
delete maps.cycleIds[name];
delete resolvedEmpty[name];
delete reasons[name];
}
});
set({
variableFetchStates: maps.states,
variableLastUpdated: maps.lastUpdated,
variableCycleIds: maps.cycleIds,
variableCycleReasons: reasons,
variableResolvedEmpty: resolvedEmpty,
variableFetchContext: context,
});
@@ -182,11 +171,6 @@ export const createVariableFetchSlice: StateCreator<
dynamicVariableOrder,
} = variableFetchContext;
const maps = cloneMaps(get());
const reasons = { ...get().variableCycleReasons };
const bump = (name: string): void => {
maps.cycleIds[name] = (maps.cycleIds[name] || 0) + 1;
reasons[name] = VariableCycleReason.FullCycle;
};
// Query variables wait only for their QUERY parents. A DYNAMIC parent does not
// gate: its option fetch feeds only its own dropdown, while its selected value
@@ -194,7 +178,7 @@ export const createVariableFetchSlice: StateCreator<
// dependent query substitutes it immediately and refetches via the cascade if
// it later changes. Text/custom parents resolve synchronously, so nothing waits.
queryVariableOrder.forEach((name) => {
bump(name);
maps.cycleIds[name] = (maps.cycleIds[name] || 0) + 1;
const parents = dependencyData.parentGraph[name] || [];
const hasQueryParents = parents.some((p) => variableTypes[p] === 'QUERY');
maps.states[name] = hasQueryParents
@@ -208,7 +192,7 @@ export const createVariableFetchSlice: StateCreator<
const orderedQuery = new Set(queryVariableOrder);
Object.keys(variableTypes).forEach((name) => {
if (variableTypes[name] === 'QUERY' && !orderedQuery.has(name)) {
bump(name);
maps.cycleIds[name] = (maps.cycleIds[name] || 0) + 1;
maps.states[name] = resolveFetchState(maps, name);
}
});
@@ -219,7 +203,7 @@ export const createVariableFetchSlice: StateCreator<
// populate fast even when query variables are slow; a sibling selection change
// later refetches them via `enqueueDescendantsBatch`.
dynamicVariableOrder.forEach((name) => {
bump(name);
maps.cycleIds[name] = (maps.cycleIds[name] || 0) + 1;
maps.states[name] = resolveFetchState(maps, name);
});
@@ -227,7 +211,6 @@ export const createVariableFetchSlice: StateCreator<
variableFetchStates: maps.states,
variableLastUpdated: maps.lastUpdated,
variableCycleIds: maps.cycleIds,
variableCycleReasons: reasons,
lastFetchAllKey: key ?? get().lastFetchAllKey,
});
},
@@ -307,11 +290,6 @@ export const createVariableFetchSlice: StateCreator<
const { dependencyData, variableTypes, dynamicVariableOrder } =
variableFetchContext;
const maps = cloneMaps(get());
const reasons = { ...get().variableCycleReasons };
const bump = (name: string): void => {
maps.cycleIds[name] = (maps.cycleIds[name] || 0) + 1;
reasons[name] = VariableCycleReason.ValueCascade;
};
const changed = new Set(names);
// Callers commit values before this runs, so the gate sees the new parent values.
const selection = selectVariableValues(get().dashboardId)(get());
@@ -327,7 +305,7 @@ export const createVariableFetchSlice: StateCreator<
});
});
queryDescendants.forEach((desc) => {
bump(desc);
maps.cycleIds[desc] = (maps.cycleIds[desc] || 0) + 1;
maps.states[desc] = queryParentsHaveValues(
desc,
variableFetchContext,
@@ -344,7 +322,7 @@ export const createVariableFetchSlice: StateCreator<
dynamicVariableOrder
.filter((dynName) => !changed.has(dynName))
.forEach((dynName) => {
bump(dynName);
maps.cycleIds[dynName] = (maps.cycleIds[dynName] || 0) + 1;
maps.states[dynName] = resolveFetchState(maps, dynName);
});
}
@@ -353,7 +331,6 @@ export const createVariableFetchSlice: StateCreator<
variableFetchStates: maps.states,
variableLastUpdated: maps.lastUpdated,
variableCycleIds: maps.cycleIds,
variableCycleReasons: reasons,
});
},
});
@@ -370,12 +347,6 @@ export const selectVariableCycleId =
(state: DashboardStore): number =>
state.variableCycleIds[name] ?? 0;
/** Selector: why a variable's cycle was enqueued. Undefined for types that never fetch. */
export const selectVariableCycleReason =
(name: string) =>
(state: DashboardStore): VariableCycleReason | undefined =>
state.variableCycleReasons[name];
/** Selector: whether a variable has completed at least one fetch. */
export const selectVariableFetchedOnce =
(name: string) =>

View File

@@ -7,14 +7,6 @@ export enum VariableFetchState {
Error = 'error',
}
/** Why a cycle was started — only a cascade may re-default a user's selection. */
export enum VariableCycleReason {
/** `enqueueFetchAll`: load, time-range or variable-order change. */
FullCycle = 'full-cycle',
/** `enqueueDescendantsBatch`: a parent or sibling variable's value changed. */
ValueCascade = 'value-cascade',
}
/** Mutable clones a fetch action works over before committing back in one `set`. */
export interface FetchMaps {
states: Record<string, VariableFetchState>;

View File

@@ -239,9 +239,16 @@ func processJSONParser(parent *pipelinetypes.PipelineOperator) ([]pipelinetypes.
return nil, errors.NewInternalf(CodeInvalidOperatorType, "operator type received %s", parent.Type)
}
// on_error: send_quiet replaces the expensive isJSON `if` check;
// parse failures pass the record through unchanged without noisy logs.
parent.OnError = signozstanzahelper.SendOnErrorQuiet
parseFromNotNilCheck, err := fieldNotNilCheck(parent.ParseFrom)
if err != nil {
return nil, errors.WrapInvalidInputf(err, CodeFieldNilCheckType,
"couldn't generate nil check for parseFrom of json parser op %s: %s", parent.Name, err,
)
}
parent.If = fmt.Sprintf(
`%s && ((type(%s) == "string" && isJSON(%s) && type(fromJSON(unquote(%s))) == "map" ) || type(%s) == "map")`,
parseFromNotNilCheck, parent.ParseFrom, parent.ParseFrom, parent.ParseFrom, parent.ParseFrom,
)
if parent.EnableFlattening {
parent.MaxFlatteningDepth = constants.MaxJSONFlatteningDepth
}
@@ -291,7 +298,7 @@ func processJSONParser(parent *pipelinetypes.PipelineOperator) ([]pipelinetypes.
}
// JSONMapping: host
err := generateMoveOperators(mapping[pipelinetypes.Host], `resource["host.name"]`)
err = generateMoveOperators(mapping[pipelinetypes.Host], `resource["host.name"]`)
if err != nil {
return nil, err
}

View File

@@ -324,17 +324,6 @@ func TestNoCollectorErrorsFromProcessorsForMismatchedLogs(t *testing.T) {
makeTestLog("mismatching log", map[string]string{
"test_json": "bad json",
}),
}, {
"json parser should quietly ignore log with non JSON body",
pipelinetypes.PipelineOperator{
ID: "json",
Type: "json_parser",
Enabled: true,
Name: "json parser",
ParseFrom: "body",
ParseTo: "attributes",
},
makeTestLog("plain text log", map[string]string{}),
}, {
"move parser should ignore non matching logs",
pipelinetypes.PipelineOperator{
@@ -905,8 +894,8 @@ func TestProcessJSONParser_WithFlatteningAndMapping(t *testing.T) {
require.Equal(t, 1, parentOp.MaxFlatteningDepth)
require.Nil(t, parentOp.Mapping) // Mapping should be removed
require.Nil(t, parent.Mapping) // Mapping should be removed
require.Empty(t, parentOp.If)
require.Equal(t, signozstanzahelper.SendOnErrorQuiet, parentOp.OnError)
require.Contains(t, parentOp.If, `isJSON(body)`)
require.Contains(t, parentOp.If, `type(body)`)
require.Equal(t, 1+totalOps, len(ops))
@@ -962,8 +951,7 @@ func TestProcessJSONParser_WithoutMapping(t *testing.T) {
require.True(t, op.EnableFlattening)
require.True(t, op.EnablePaths)
require.Equal(t, "parsed", op.PathPrefix)
require.Empty(t, op.If)
require.Equal(t, signozstanzahelper.SendOnErrorQuiet, op.OnError)
require.Contains(t, op.If, `isJSON(body)`)
}
func TestProcessJSONParser_Simple(t *testing.T) {
@@ -987,8 +975,7 @@ func TestProcessJSONParser_Simple(t *testing.T) {
require.False(t, op.EnableFlattening)
require.False(t, op.EnablePaths)
require.Equal(t, "", op.PathPrefix)
require.Empty(t, op.If)
require.Equal(t, signozstanzahelper.SendOnErrorQuiet, op.OnError)
require.Contains(t, op.If, `isJSON(body)`)
}
func TestProcessJSONParser_InvalidType(t *testing.T) {

View File

@@ -56,6 +56,17 @@ func QueryStringToKeysSelectors(query string) []*telemetrytypes.FieldKeySelector
FieldDataType: key.FieldDataType,
})
}
// todo(tushar): consider reverting changes done to this method in below PR to avoid scope specific checks
// https://github.com/SigNoz/signoz/issues/11374
if key.FieldContext == telemetrytypes.FieldContextScope {
keys = append(keys, &telemetrytypes.FieldKeySelector{
Name: key.FieldContext.StringValue() + "." + key.Name,
Signal: key.Signal,
FieldContext: telemetrytypes.FieldContextUnspecified, // this allows 'scope.' prefix for keys with other context as well
FieldDataType: key.FieldDataType,
})
}
}
}

View File

@@ -72,6 +72,23 @@ func TestQueryToKeys(t *testing.T) {
},
},
},
{
query: `scope.version = '1.0.0'`,
expectedKeys: []telemetrytypes.FieldKeySelector{
{
Name: "version",
Signal: telemetrytypes.SignalUnspecified,
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
{
Name: "scope.version",
Signal: telemetrytypes.SignalUnspecified,
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
},
},
}
for _, testCase := range testCases {

View File

@@ -373,6 +373,94 @@ func TestStatementBuilder(t *testing.T) {
},
expectedErr: nil,
},
{
name: "scope.name filter and group by",
requestType: qbtypes.RequestTypeTimeSeries,
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.TraceAggregation{
{
Expression: "count()",
},
},
Filter: &qbtypes.Filter{
Expression: "scope.name = 'opentelemetry-io'",
},
Limit: 10,
GroupBy: []qbtypes.GroupByKey{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "scope.name",
FieldContext: telemetrytypes.FieldContextScope,
},
},
},
},
expected: qbtypes.Statement{
Query: "WITH __limit_cte AS (SELECT toString(multiIf(scope.name::String IS NOT NULL, scope.name::String, NULL)) AS `scope.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.name::String = ? AND scope.name::String IS NOT NULL) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `scope.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(scope.name::String IS NOT NULL, scope.name::String, NULL)) AS `scope.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.name::String = ? AND scope.name::String IS NOT NULL) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`scope.name`) GLOBAL IN (SELECT `scope.name` FROM __limit_cte) GROUP BY ts, `scope.name`",
Args: []any{"opentelemetry-io", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "opentelemetry-io", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
expectedErr: nil,
},
{
name: "scope.version filter with scope.name group by",
requestType: qbtypes.RequestTypeTimeSeries,
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.TraceAggregation{
{
Expression: "count()",
},
},
Filter: &qbtypes.Filter{
Expression: "scope.version = '1.0.0'",
},
Limit: 10,
GroupBy: []qbtypes.GroupByKey{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "scope.name",
FieldContext: telemetrytypes.FieldContextScope,
},
},
},
},
expected: qbtypes.Statement{
Query: "WITH __limit_cte AS (SELECT toString(multiIf(scope.name::String IS NOT NULL, scope.name::String, NULL)) AS `scope.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.version::String = ? AND scope.version::String IS NOT NULL) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `scope.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(scope.name::String IS NOT NULL, scope.name::String, NULL)) AS `scope.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.version::String = ? AND scope.version::String IS NOT NULL) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`scope.name`) GLOBAL IN (SELECT `scope.name` FROM __limit_cte) GROUP BY ts, `scope.name`",
Args: []any{"1.0.0", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1.0.0", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
expectedErr: nil,
},
{
name: "scope.version filter only (no scope field in group by)",
requestType: qbtypes.RequestTypeTimeSeries,
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.TraceAggregation{
{
Expression: "count()",
},
},
Filter: &qbtypes.Filter{
Expression: "scope.version = '1.0.0'",
},
Limit: 10,
GroupBy: []qbtypes.GroupByKey{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
},
},
},
},
expected: qbtypes.Statement{
Query: "WITH __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.version::String = ? AND scope.version::String IS NOT NULL) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.version::String = ? AND scope.version::String IS NOT NULL) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
Args: []any{"1.0.0", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1.0.0", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
},
}
fl := flaggertest.New(t)
@@ -799,6 +887,52 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
},
expectedErr: nil,
},
{
name: "List query with scope filter only (no scope in select or group by)",
requestType: qbtypes.RequestTypeRaw,
keysMap: map[string][]*telemetrytypes.TelemetryFieldKey{
"scope.version": {
{
Name: "scope.version",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
},
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Filter: &qbtypes.Filter{
Expression: "scope.version = '1.0.0'",
},
Limit: 10,
},
expected: qbtypes.Statement{
Query: "SELECT timestamp AS `timestamp`, trace_id AS `trace_id`, span_id AS `span_id`, trace_state AS `trace_state`, parent_span_id AS `parent_span_id`, flags AS `flags`, name AS `name`, kind AS `kind`, kind_string AS `kind_string`, duration_nano AS `duration_nano`, status_code AS `status_code`, status_message AS `status_message`, status_code_string AS `status_code_string`, events AS `events`, links AS `links`, response_status_code AS `response_status_code`, external_http_url AS `external_http_url`, http_url AS `http_url`, external_http_method AS `external_http_method`, http_method AS `http_method`, http_host AS `http_host`, db_name AS `db_name`, db_operation AS `db_operation`, has_error AS `has_error`, is_remote AS `is_remote`, attributes_string, attributes_number, attributes_bool, resources_string FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.version::String = ? AND scope.version::String IS NOT NULL) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"1.0.0", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10},
},
},
{
// Regression test: scope.version in selectFields with no metadata (isColumn=true filters it out)
// must still produce scope.version::String, not scope.attributes.version::String
name: "scope.version in selectFields only, no metadata (intrinsic field fallback)",
requestType: qbtypes.RequestTypeRaw,
keysMap: map[string][]*telemetrytypes.TelemetryFieldKey{},
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Filter: &qbtypes.Filter{},
SelectFields: []telemetrytypes.TelemetryFieldKey{
{Name: "scope.version", FieldContext: telemetrytypes.FieldContextUnspecified},
},
Limit: 10,
},
expected: qbtypes.Statement{
Query: "SELECT timestamp AS `timestamp`, trace_id AS `trace_id`, span_id AS `span_id`, scope.version::String AS `scope.version` FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10},
},
},
}
for _, c := range cases {

View File

@@ -180,7 +180,7 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
`CASE
// WHEN tagType = 'spanfield' THEN 1
WHEN tagType = 'resource' THEN 2
// WHEN tagType = 'scope' THEN 3
WHEN tagType = 'scope' THEN 3
WHEN tagType = 'tag' THEN 4
ELSE 5
END as priority`,

View File

@@ -121,6 +121,20 @@ var (
FieldContext: telemetrytypes.FieldContextSpan,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
"scope.name": {
Name: "scope.name",
Description: "Instrumentation scope name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
"scope.version": {
Name: "scope.version",
Description: "Instrumentation scope version",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
}
IntrinsicFieldsDeprecated = map[string]telemetrytypes.TelemetryFieldKey{
"traceID": {

View File

@@ -52,6 +52,7 @@ var (
ValueType: schema.ColumnTypeString,
}},
"resource": {Name: "resource", Type: schema.JSONColumnType{}},
"scope": {Name: "scope", Type: schema.JSONColumnType{}},
"events": {Name: "events", Type: schema.ArrayColumnType{
ElementType: schema.ColumnTypeString,
@@ -176,7 +177,7 @@ func (m *fieldMapper) getColumn(
case telemetrytypes.FieldContextResource:
return []*schema.Column{indexV3Columns["resource"], indexV3Columns["resources_string"]}, nil
case telemetrytypes.FieldContextScope:
return []*schema.Column{}, qbtypes.ErrColumnNotFound
return []*schema.Column{indexV3Columns["scope"]}, nil
case telemetrytypes.FieldContextAttribute:
switch key.FieldDataType {
case telemetrytypes.FieldDataTypeString:
@@ -287,14 +288,24 @@ func (m *fieldMapper) resolveColumnExprs(
switch column.Type.GetType() {
case schema.ColumnTypeEnumJSON:
// json is only supported for resource context as of now
if key.FieldContext != telemetrytypes.FieldContextResource {
return nil, nil, nil, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns, got %s", key.FieldContext.String)
}
// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
// once clickHouse dependency is updated, we need to check if we can remove it.
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
existExprs = append(existExprs, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
switch key.FieldContext {
case telemetrytypes.FieldContextResource:
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
existExprs = append(existExprs, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
case telemetrytypes.FieldContextScope:
switch key.Name {
case "scope.name", "scope.version":
exprs = append(exprs, fmt.Sprintf("%s::String", key.Name))
existExprs = append(existExprs, fmt.Sprintf("%s IS NOT NULL", key.Name))
default:
exprs = append(exprs, fmt.Sprintf("%s.attributes.`%s`::String", columnName, key.Name))
existExprs = append(existExprs, fmt.Sprintf("%s.attributes.`%s` IS NOT NULL", columnName, key.Name))
}
default:
return nil, nil, nil, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource and scope context fields are supported for json columns, got %s", key.FieldContext.String)
}
case schema.ColumnTypeEnumString,
schema.ColumnTypeEnumUInt64,
schema.ColumnTypeEnumUInt32,

View File

@@ -83,6 +83,33 @@ func TestGetFieldKeyName(t *testing.T) {
expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`, `resource_string_deployment$$environment`, NULL)",
expectedError: nil,
},
{
name: "Scope field - scope.name",
key: telemetrytypes.TelemetryFieldKey{
Name: "scope.name",
FieldContext: telemetrytypes.FieldContextScope,
},
expectedResult: "scope.name::String",
expectedError: nil,
},
{
name: "Scope field - scope.version",
key: telemetrytypes.TelemetryFieldKey{
Name: "scope.version",
FieldContext: telemetrytypes.FieldContextScope,
},
expectedResult: "scope.version::String",
expectedError: nil,
},
{
name: "Scope field - custom attribute",
key: telemetrytypes.TelemetryFieldKey{
Name: "custom.attr",
FieldContext: telemetrytypes.FieldContextScope,
},
expectedResult: "scope.attributes.`custom.attr`::String",
expectedError: nil,
},
{
// Query like `attribute.attribute_string:string` should resolve to `attributes_string['attribute_string']`.
name: "Attribute key whose name collides with contextual map column resolves as a map lookup",

View File

@@ -113,6 +113,20 @@ func BuildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytype
FieldDataType: telemetrytypes.FieldDataTypeBool,
},
},
"scope.name": {
{
Name: "scope.name",
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
"scope.version": {
{
Name: "scope.version",
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
}
for _, keys := range keysMap {
for _, key := range keys {

View File

@@ -995,6 +995,8 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"cloud.provider": "integration",
"cloud.account.id": "000",
"trace_id": "corrupt_data",
"scope_name": "corrupt_data",
"scope.scope.name": "corrupt_data",
},
attributes={
"net.transport": "IP.TCP",
@@ -1003,7 +1005,10 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"http.request.method": "POST",
"http.response.status_code": "200",
"timestamp": "corrupt_data",
"version": "1.0.0",
"scope.scope.version": "1.0.0",
},
scope={"name": "io.signoz.http.server", "version": "2.0.0"},
),
Traces(
timestamp=now - timedelta(seconds=3.5),
@@ -1023,12 +1028,24 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"cloud.provider": "integration",
"cloud.account.id": "000",
"timestamp": "corrupt_data",
"scope.attributes.name": "corrupt_data",
},
attributes={
"db.name": "integration",
"db.operation": "SELECT",
"db.statement": "SELECT * FROM integration",
"trace_d": "corrupt_data",
"scope.attributes.version": "corrupt_data",
},
scope={
"name": "io.opentelemetry.contrib.http",
"version": "1.0.0",
"attributes": {
"telemetry.sdk.language": "cpp",
"name": "not-the-real-name",
"version": "not-the-real-version",
"attributes": "literally-a-key-named-attributes",
},
},
),
Traces(
@@ -1049,12 +1066,15 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"cloud.provider": "integration",
"cloud.account.id": "000",
"duration_nano": "corrupt_data",
"scope.scope.attributes.version": "corrupt_data",
},
attributes={
"http.request.method": "PATCH",
"http.status_code": "404",
"id": "1",
"scope.scope.version": "corrupt_data",
},
scope={"name": "io.signoz.http.client", "version": "2.0.0"},
),
Traces(
timestamp=now - timedelta(seconds=1),
@@ -1073,6 +1093,7 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"host.name": "linux-001",
"cloud.provider": "integration",
"cloud.account.id": "001",
"scope.scope.version": "corrupt_data",
},
attributes={
"message.type": "SENT",
@@ -1080,7 +1101,10 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"messaging.message.id": "001",
"duration_nano": "corrupt_data",
"id": 1,
"scope": "corrupt_data",
"scope.attributes.name": "corrupt_data",
},
scope={"name": "io.signoz.messaging", "version": "3.0.0"},
),
]

View File

@@ -302,6 +302,7 @@ class Traces(ABC):
db_operation: str
has_error: bool
is_remote: str
scope_json: dict[str, Any]
resource: list[TracesResource]
tag_attributes: list[TracesTagAttributes]
@@ -327,6 +328,7 @@ class Traces(ABC):
links: list[TracesLink] = [],
trace_state: str = "",
flags: np.uint32 = 0,
scope: dict[str, Any] = {},
resource_write_mode: Literal["legacy_only", "dual_write"] = "dual_write",
) -> None:
if timestamp is None:
@@ -408,6 +410,33 @@ class Traces(ABC):
# Calculate resource fingerprint
self.resource_fingerprint = LogsOrTracesFingerprint(self.resources_string).calculate()
# Process scope mirroring the InstrumentationScope on the OTLP span.
scope_name = scope.get("name", "")
scope_version = scope.get("version", "")
scope_string = {k: str(v) for k, v in scope.get("attributes", {}).items()}
self.scope_json = {
"name": scope_name,
"version": scope_version,
"attributes": scope_string,
}
scope_keys = {"scope.name": scope_name, "scope.version": scope_version}
scope_keys.update(scope_string)
for k, v in scope_keys.items():
if v == "":
continue
self.tag_attributes.append(
TracesTagAttributes(
timestamp=timestamp,
tag_key=k,
tag_type="scope",
tag_data_type="string",
string_value=v,
number_value=None,
)
)
self.attribute_keys.append(TracesResourceOrAttributeKeys(name=k, datatype="string", tag_type="scope"))
# Process attributes by type and populate custom fields
self.attribute_string = {}
self.attributes_number = {}
@@ -660,6 +689,7 @@ class Traces(ABC):
self.has_error,
self.is_remote,
self.resource_json,
self.scope_json,
],
dtype=object,
)
@@ -691,6 +721,7 @@ class Traces(ABC):
attributes=data.get("attributes", {}),
trace_state=data.get("trace_state", ""),
flags=data.get("flags", 0),
scope=data.get("scope", {}),
)
@classmethod
@@ -830,6 +861,7 @@ def insert_traces_to_clickhouse(conn, traces: list[Traces]) -> None:
"has_error",
"is_remote",
"resource",
"scope",
],
data=[trace.np_arr() for trace in traces],
)

View File

@@ -359,28 +359,15 @@ def test_preview_logs_pipelines_success(
) -> None:
"""
Setup:
Preview a json_parser pipeline with one JSON log and one plain-text log.
Create a preview request with a pipeline and sample logs.
Tests:
1. JSON body gets parsed into attributes
2. Non-JSON body passes through unchanged instead of being dropped
1. Send preview request with valid pipeline configuration
2. Verify the preview processes logs correctly
3. Verify the response contains processed logs
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
empty_log_fields = {
"id": "",
"trace_id": "",
"span_id": "",
"trace_flags": 0,
"severity_text": "",
"severity_number": 0,
"attributes_string": {},
"attributes_int": {},
"attributes_float": {},
"attributes_bool": {},
"resources_string": {},
}
preview_payload = {
"pipelines": [
{
@@ -409,25 +396,29 @@ def test_preview_logs_pipelines_success(
{
"type": "json_parser",
"id": "json-parser-preview",
"orderId": 1,
"enabled": True,
"parse_from": "body",
"parse_to": "attributes",
"on_error": "send",
}
],
}
],
"logs": [
{
"body": '{"level": "info", "message": "json log"}',
"body": '{"level": "info", "message": "Test log message", "timestamp": "2024-01-01T00:00:00Z"}',
"timestamp": 1704067200000000000, # nanoseconds, not milliseconds
**empty_log_fields,
},
{
"body": "plain text log that is not json",
"timestamp": 1704067201000000000,
**empty_log_fields,
},
"id": "",
"trace_id": "",
"span_id": "",
"trace_flags": 0,
"severity_text": "",
"severity_number": 0,
"attributes_string": {},
"attributes_int": {},
"attributes_float": {},
"attributes_bool": {},
"resources_string": {"service.name": "test-service"},
}
],
}
@@ -444,16 +435,13 @@ def test_preview_logs_pipelines_success(
assert response.status_code == HTTPStatus.OK
response_data = response.json()
assert response_data["status"] == "success"
logs = response_data["data"]["logs"]
assert len(logs) == 2
assert "data" in response_data
assert "logs" in response_data["data"]
assert len(response_data["data"]["logs"]) == 1
json_log = next(log for log in logs if log["body"].startswith("{"))
assert json_log["attributes_string"]["level"] == "info"
assert json_log["attributes_string"]["message"] == "json log"
plain_log = next(log for log in logs if not log["body"].startswith("{"))
assert plain_log["body"] == "plain text log that is not json"
assert plain_log["attributes_string"] == {}
# Verify the log was processed
processed_log = response_data["data"]["logs"][0]
assert "attributes_string" in processed_log or "attributes" in processed_log
def test_create_multiple_pipelines_success(

View File

@@ -1199,6 +1199,13 @@ def test_traces_list_span_scope(
lambda x: {"duration_nano": int(x[1].duration_nano), "span_id": x[1].span_id, "timestamp": format_timestamp(x[1].timestamp), "trace_id": x[1].trace_id},
id="select_attribute_duration_order_intrinsic",
),
# Case 9: filter on the intrinsic scope.version. Only x[1] should match.
pytest.param(
BuilderQuery(signal="traces", name="A", select_fields=[TelemetryFieldKey("timestamp")], filter_expression="scope.version = '1.0.0'", limit=1),
HTTPStatus.OK,
lambda x: {"span_id": x[1].span_id, "timestamp": format_timestamp(x[1].timestamp), "trace_id": x[1].trace_id},
id="filter_scope_version",
),
],
)
def test_traces_list_with_corrupt_data(
@@ -1242,6 +1249,156 @@ def test_traces_list_with_corrupt_data(
assert get_rows(response)[0]["data"] == expected(traces)
@pytest.mark.parametrize(
"filter_expression,expected_indices",
[
# Intrinsic scope.name / scope.version resolve to the JSON sub-columns.
pytest.param("scope.name = 'io.signoz.payment'", [1], id="intrinsic_scope_name"),
pytest.param("scope.version = '2.3.1'", [0], id="intrinsic_scope_version"),
# A scope attribute resolves against the scope JSON column's attributes.
pytest.param("scope.telemetry.sdk.language = 'python'", [1], id="scope_attribute"),
# `env.tier` is a span attribute on span 0 and a scope attribute on
# span 1. Unprefixed -> no explicit context, so it is checked in every
# applicable context (attribute OR scope) and both spans match.
pytest.param("env.tier = 'gold'", [0, 1], id="bare_cross_context"),
# The explicit `scope.` prefix forces scope context only, so span 0's
# span attribute is ignored — only span 1 matches.
pytest.param("scope.env.tier = 'gold'", [1], id="scope_prefixed_cross_context"),
# `scope.name` matches BOTH the intrinsic scope.name field (span 0) and a
# scope attribute literally named `name` (span 1's scope attribute
# name='io.signoz.checkout').
pytest.param("scope.name = 'io.signoz.checkout'", [0, 1], id="scope_name_collision"),
# `scope.name` also matches a span attribute literally named `scope.name`
# (attribute context) — span 2 carries attribute scope.name='attr-scope-name'.
pytest.param("scope.name = 'attr-scope-name'", [2], id="scope_name_attribute_collision"),
# An unprefixed `name` resolves to the intrinsic span `name` column and a
# `name` scope attribute, but NOT the scope.name field. Span 2's span
# name and span 1's scope attribute `name` both equal 'io.signoz.checkout';
# span 0's scope.name field equals it too but is NOT matched.
pytest.param("name = 'io.signoz.checkout'", [1, 2], id="bare_name_excludes_scope_name_field"),
# A value that no resolvable key holds (scope.name/scope.version field,
# a `name`/`version` scope attribute, or a same-named attribute/resource)
# returns nothing.
pytest.param("scope.version = 'corrupt_data'", [], id="scope_version_no_match"),
pytest.param("scope.name = 'corrupt_data'", [], id="scope_name_no_match"),
],
)
def test_traces_list_with_scope_filter(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
filter_expression: str,
expected_indices: list[int],
) -> None:
"""
Setup three spans with different scope key resolution:
- x[0]: scope.name/version 'io.signoz.checkout'/'2.3.1'; span attribute
env.tier='gold'.
- x[1]: scope.name/version 'io.signoz.payment'/'4.5.6'; scope attributes
telemetry.sdk.language='python', env.tier='gold', and a `name` scope
attribute colliding with x[0]'s scope.name value.
- x[2]: span name 'io.signoz.checkout' (colliding with x[0]'s scope.name
value) and a span attribute literally named `scope.name`.
Tests:
- Filtering on scope.name / scope.version / a scope attribute.
- An unprefixed key is resolved across contexts (scope checked alongside
attribute / intrinsic), while a `scope.`-prefixed key is scope-only.
- `scope.name` hits the intrinsic field, a `name` scope attribute, and a
span attribute `scope.name` (cross-context), while a bare `name` hits
the span name column (and a `name` scope attribute) but never the
scope.name field.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
trace_id = TraceIdGenerator.trace_id()
span_ids = [TraceIdGenerator.span_id() for _ in range(3)]
traces = [
Traces(
timestamp=now - timedelta(seconds=4),
duration=timedelta(seconds=2),
trace_id=trace_id,
span_id=span_ids[0],
parent_span_id="",
name="GET /checkout",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
resources={"service.name": "checkout"},
attributes={"http.request.method": "GET", "env.tier": "gold"},
scope={
"name": "io.signoz.checkout",
"version": "2.3.1",
"attributes": {"telemetry.sdk.language": "go"},
},
),
Traces(
timestamp=now - timedelta(seconds=2),
duration=timedelta(seconds=1),
trace_id=trace_id,
span_id=span_ids[1],
parent_span_id="",
name="POST /pay",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
resources={"service.name": "payment"},
attributes={"http.request.method": "POST"},
# env.tier is a scope attribute here (cross-context with span 0);
# `name` is a scope attribute colliding with span 0's scope.name.
scope={
"name": "io.signoz.payment",
"version": "4.5.6",
"attributes": {
"telemetry.sdk.language": "python",
"env.tier": "gold",
"name": "io.signoz.checkout",
},
},
),
Traces(
timestamp=now - timedelta(seconds=1),
duration=timedelta(seconds=1),
trace_id=trace_id,
span_id=span_ids[2],
parent_span_id="",
# span name collides with span 0's scope.name value
name="io.signoz.checkout",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
resources={"service.name": "probe"},
# a span attribute named `scope.name`
attributes={"scope.name": "attr-scope-name"},
scope={"name": "span-gamma", "version": "9.9.9"},
),
]
insert_traces(traces)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
start_ms, end_ms = _query_window(now)
response = make_query_request(
signoz,
token,
start_ms=start_ms,
end_ms=end_ms,
request_type=RequestType.RAW,
queries=[
BuilderQuery(
signal="traces",
name="A",
select_fields=[TelemetryFieldKey("timestamp")],
filter_expression=filter_expression,
limit=10,
).to_dict()
],
)
assert response.status_code == HTTPStatus.OK, response.text
got_span_ids = {row["data"]["span_id"] for row in get_rows(response)}
expected_span_ids = {traces[i].span_id for i in expected_indices}
assert got_span_ids == expected_span_ids
@pytest.mark.parametrize("surface", ["filter", "select", "order"])
def test_traces_list_unknown_span_context_synthesizes(
signoz: types.SigNoz,