Compare commits

..

5 Commits

Author SHA1 Message Date
Ashwin Bhatkal
fb326a9146 fix(dashboard-v2): a no-op close of the variable list commits nothing
The shared control reads a selection covering every option as ALL, so opening and
closing the dropdown without touching it promoted an explicit pick into a standing
ALL whenever the current window offered only the selected values — and rewrote a
dynamic ALL's `__all__` into concrete values. A close that left the list as it
opened now commits nothing.

The commit rule (fallback when empty, ALL when exactly the options, typed values
marked) moves to a resolver in utils, out of the component.
2026-08-05 21:40:15 +05:30
Ashwin Bhatkal
ed287be741 fix(dashboard-v2): keep typed-in variable values through every refetch
A value typed into a multi-select is in no option list, so the reconcile read it as
"no longer offered" and dropped it on the next refetch — of any cause. The selection
now records which entries were typed, judged at pick time against the options then
offered, and those are never dropped.

ALL also means exactly the option set now: a set that additionally carries a typed
value is not ALL, since ALL re-materializes to the options alone and would take the
typed value with it on the next refetch.
2026-08-05 21:40:04 +05:30
Ashwin Bhatkal
52d1731d48 fix(dashboard-v2): keep a variable's selection across a time-range refetch
Changing the time range refetches every variable's options. A window with no data
for the selected values left the multi-select reconcile with nothing valid, so it
re-defaulted — which, since #12343, means ALL for an ALL-enabled variable. Picking
one service and switching to a quieter window silently widened every panel on the
dashboard to all services.

The reconcile now re-defaults only when another variable's value re-scoped the
options; a refetch nothing else caused leaves the pick alone. Single-select has
behaved this way since #12178 — this is the multi-select half of the same rule,
which was left untouched then and untested, so the regression shipped green.
2026-08-05 21:39:44 +05:30
Ashwin Bhatkal
feeee00791 refactor(dashboard-v2): record why each variable fetch cycle was enqueued
A cycle starts either because everything refetches (load, time-range or
variable-order change) or because another variable's value re-scoped this one's
options. The engine knew which, but discarded it — so the post-fetch reconcile
could not tell "the options moved under me" from "my options were re-scoped".
Tag it per variable, alongside the cycle id it is bumped with.
2026-08-05 21:39:29 +05:30
Nityananda Gohain
7633845f27 fix: remove if condition for json parser (#12407)
* fix: remove if condition for json parser

* fix: update integration tests
2026-08-05 12:37:42 +00:00
28 changed files with 633 additions and 519 deletions

3
.gitignore vendored
View File

@@ -231,5 +231,4 @@ cython_debug/
# LSP config files
pyrightconfig.json
# agents
.claude/settings.local.json

View File

@@ -182,4 +182,56 @@ describe('ValueSelector', () => {
});
});
});
describe('opening and closing without touching the list', () => {
function renderWith(
selection: VariableSelection,
options: string[],
): jest.Mock {
const onChange = jest.fn();
render(
<TooltipProvider>
<ValueSelector
options={options}
variableType="dynamic"
multiSelect
showAllOption
selection={selection}
onChange={onChange}
emptyFallback={{ value: [], allSelected: false }}
testId="variable-select-env"
/>
</TooltipProvider>,
);
return onChange;
}
async function openThenClose(): Promise<void> {
const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime });
const control = screen.getByTestId('variable-select-env');
await user.click(control.querySelector('input') as HTMLInputElement);
await user.keyboard('{Escape}');
}
it('does not promote a pick that covers every available option to ALL', async () => {
// A narrow time range can leave only the selected value in the list. That is
// still an explicit pick, not "everything, always".
const onChange = renderWith(
{ value: ['checkout-service-prod'], allSelected: false },
['checkout-service-prod'],
);
await openThenClose();
expect(onChange).not.toHaveBeenCalled();
});
it('does not rewrite a dynamic ALL into concrete values', async () => {
const onChange = renderWith({ value: null, allSelected: true }, OPTIONS);
await openThenClose();
expect(onChange).not.toHaveBeenCalled();
});
});
});

View File

@@ -145,6 +145,133 @@ describe('reconcileWithOptions', () => {
),
).toBeNull();
});
describe('preserveSelection (options moved on their own — time range, reload)', () => {
const multi = model({
type: 'DYNAMIC',
multiSelect: true,
showAllOption: true,
dynamicAttribute: 'service.name',
});
it('keeps a multi-select pick the new option list no longer offers', () => {
expect(
reconcileWithOptions(multi, { value: ['frontend'], allSelected: false }, [
'backend',
'cart',
]),
).toStrictEqual({ value: null, allSelected: true });
expect(
reconcileWithOptions(
multi,
{ value: ['frontend'], allSelected: false },
['backend', 'cart'],
{ preserveSelection: true },
),
).toBeNull();
});
it('still materializes ALL, which must track the option list', () => {
expect(
reconcileWithOptions(
model({ type: 'QUERY', multiSelect: true, showAllOption: true }),
{ value: ['a'], allSelected: true },
['a', 'b'],
{ preserveSelection: true },
),
).toStrictEqual({ value: ['a', 'b'], allSelected: true });
});
it('still fills the default when nothing is selected yet', () => {
expect(
reconcileWithOptions(multi, { value: [], allSelected: false }, ['a', 'b'], {
preserveSelection: true,
}),
).toStrictEqual({ value: null, allSelected: true });
});
});
// A typed value is in no option list, so no refetch can invalidate it.
describe('customValues (typed in, never offered by the data)', () => {
const multi = model({
type: 'DYNAMIC',
multiSelect: true,
showAllOption: true,
dynamicAttribute: 'service.name',
});
it('keeps them through a re-scope that drops a fetched value', () => {
expect(
reconcileWithOptions(
multi,
{
value: ['frontend', 'typed-in'],
allSelected: false,
customValues: ['typed-in'],
},
['backend', 'cart'],
),
).toStrictEqual({
value: ['typed-in'],
allSelected: false,
customValues: ['typed-in'],
});
});
it('never re-defaults a selection made only of them', () => {
expect(
reconcileWithOptions(
multi,
{ value: ['typed-in'], allSelected: false, customValues: ['typed-in'] },
['backend', 'cart'],
),
).toBeNull();
});
// An inert marker is not worth a store write + dependent refetch to prune.
it('leaves a stale marker alone when it drops nothing', () => {
expect(
reconcileWithOptions(
multi,
{
value: ['frontend', 'typed-in'],
allSelected: false,
customValues: ['typed-in', 'removed-earlier'],
},
['frontend'],
),
).toBeNull();
});
it('prunes markers for values it does drop', () => {
expect(
reconcileWithOptions(
multi,
{
value: ['stale', 'typed-in'],
allSelected: false,
customValues: ['typed-in'],
},
['frontend'],
),
).toStrictEqual({
value: ['typed-in'],
allSelected: false,
customValues: ['typed-in'],
});
});
it('still drops an unmarked value the list no longer offers', () => {
expect(
reconcileWithOptions(
multi,
{ value: ['frontend', 'stale'], allSelected: false },
['frontend'],
),
).toStrictEqual({ value: ['frontend'], allSelected: false });
});
});
});
describe('configuredDefaultValue', () => {

View File

@@ -0,0 +1,91 @@
import type { VariableSelection } from '../selectionTypes';
import { selectionFromCommittedValues } from '../utils/selectionUtils';
const OPTIONS = ['checkout', 'payments', 'cart'];
const FALLBACK: VariableSelection = { value: null, allSelected: true };
function commit(
values: string[],
overrides: Partial<Parameters<typeof selectionFromCommittedValues>[0]> = {},
): VariableSelection {
return selectionFromCommittedValues({
values,
options: OPTIONS,
showAllOption: true,
emptyFallback: FALLBACK,
...overrides,
});
}
// What a multi-select commit resolves to. The option list is known only here, so this
// is the one place a typed value can be recognised.
describe('selectionFromCommittedValues', () => {
it('marks values the option list did not offer as typed in', () => {
expect(commit(['checkout', 'typed-in'])).toStrictEqual({
value: ['checkout', 'typed-in'],
allSelected: false,
customValues: ['typed-in'],
});
});
it('marks a selection made only of typed-in values', () => {
expect(commit(['a', 'b'])).toStrictEqual({
value: ['a', 'b'],
allSelected: false,
customValues: ['a', 'b'],
});
});
it('records no marker when every pick came from the list', () => {
expect(commit(['checkout', 'cart'])).toStrictEqual({
value: ['checkout', 'cart'],
allSelected: false,
});
});
it('reads a set covering every option as ALL', () => {
expect(commit(OPTIONS)).toStrictEqual({
value: OPTIONS,
allSelected: true,
});
});
// ALL re-materializes to the option set, so recording this as ALL would drop the
// typed value on the next refetch.
it('does not read every option PLUS a typed value as ALL', () => {
expect(commit([...OPTIONS, 'typed-in'])).toStrictEqual({
value: [...OPTIONS, 'typed-in'],
allSelected: false,
customValues: ['typed-in'],
});
});
// Derived from the values + options at commit time, never from the old selection.
it('recomputes the marker: a typed value the data now offers is a normal pick', () => {
expect(
commit(['checkout', 'was-typed'], {
options: [...OPTIONS, 'was-typed'],
}),
).toStrictEqual({ value: ['checkout', 'was-typed'], allSelected: false });
});
it('does not read it as ALL when the variable offers no ALL', () => {
expect(commit(OPTIONS, { showAllOption: false })).toStrictEqual({
value: OPTIONS,
allSelected: false,
});
});
it('resolves an empty commit to the variable fallback', () => {
expect(commit([])).toBe(FALLBACK);
});
it('marks everything while the options have not arrived', () => {
// Nothing to judge against yet; erring this way keeps a value rather than dropping it.
expect(commit(['typed-in'], { options: [] })).toStrictEqual({
value: ['typed-in'],
allSelected: false,
customValues: ['typed-in'],
});
});
});

View File

@@ -4,6 +4,8 @@ import {
emptyVariableFormModel,
type VariableFormModel,
} from '../../DashboardSettings/Variables/variableFormModel';
import { VariableCycleReason } from '../../store/slices/variableFetchSlice';
import { useDashboardStore } from '../../store/useDashboardStore';
import type { VariableSelection } from '../selectionTypes';
import { useAutoSelect } from '../hooks/useAutoSelect';
@@ -15,7 +17,11 @@ function run(
variable: VariableFormModel,
options: string[],
selection: VariableSelection,
cycleReason?: VariableCycleReason,
): VariableSelection | undefined {
useDashboardStore.setState({
variableCycleReasons: cycleReason ? { [variable.name]: cycleReason } : {},
});
const onAutoSelect = jest.fn();
renderHook(() => useAutoSelect(variable, options, selection, onAutoSelect));
return onAutoSelect.mock.calls[0]?.[0];
@@ -70,11 +76,13 @@ describe('useAutoSelect', () => {
expect(next).toStrictEqual({ value: ['a', 'b'], allSelected: true });
});
it('falls back to ALL, not the first option, when every selected value is gone', () => {
// Re-scoped options only — a time-range refetch must NOT re-default; see below.
it('re-scoped: falls back to ALL, not the first option, when every selected value is gone', () => {
const next = run(
model({ type: 'QUERY', multiSelect: true, showAllOption: true }),
['x', 'y'],
{ value: ['a', 'b'], allSelected: false },
VariableCycleReason.ValueCascade,
);
expect(next).toStrictEqual({ value: ['x', 'y'], allSelected: true });
});
@@ -102,20 +110,23 @@ describe('useAutoSelect', () => {
expect(next).toStrictEqual({ value: ['b'], allSelected: false });
});
it('keeps the still-valid subset of a multi-select when options re-scope', () => {
it('re-scoped: keeps the still-valid subset of a multi-select', () => {
const next = run(
model({ type: 'QUERY', multiSelect: true }),
['a', 'b', 'd'],
{ value: ['a', 'b', 'c'], allSelected: false },
VariableCycleReason.ValueCascade,
);
expect(next).toStrictEqual({ value: ['a', 'b'], allSelected: false });
});
it('re-defaults a multi-select when none of the selected values remain', () => {
const next = run(model({ type: 'QUERY', multiSelect: true }), ['x', 'y'], {
value: ['a', 'b'],
allSelected: false,
});
it('re-scoped: re-defaults a multi-select when none of the selected values remain', () => {
const next = run(
model({ type: 'QUERY', multiSelect: true }),
['x', 'y'],
{ value: ['a', 'b'], allSelected: false },
VariableCycleReason.ValueCascade,
);
expect(next).toStrictEqual({ value: ['x'], allSelected: false });
});
@@ -151,4 +162,45 @@ describe('useAutoSelect', () => {
});
expect(next).toBeUndefined();
});
describe('by cycle reason', () => {
const service = model({
name: 'service',
type: 'DYNAMIC',
multiSelect: true,
showAllOption: true,
dynamicAttribute: 'service.name',
});
const gone: VariableSelection = { value: ['frontend'], allSelected: false };
it('keeps the selection when a full cycle refetched the options', () => {
// The new window has no data for the selected service — no reason to widen to ALL.
const next = run(
service,
['backend', 'cart'],
gone,
VariableCycleReason.FullCycle,
);
expect(next).toBeUndefined();
});
it('re-scopes the selection when a value cascade refetched the options', () => {
const next = run(
service,
['backend', 'cart'],
gone,
VariableCycleReason.ValueCascade,
);
expect(next).toStrictEqual({ value: null, allSelected: true });
});
it('reconciles a variable with no cycle of its own (custom definition change)', () => {
const next = run(
model({ name: 'env', type: 'CUSTOM', multiSelect: true }),
['staging', 'prod'],
{ value: ['dev'], allSelected: false },
);
expect(next).toStrictEqual({ value: ['staging'], allSelected: false });
});
});
});

View File

@@ -13,11 +13,11 @@ jest.mock('nuqs', () => ({
useQueryState: (): unknown => [null, jest.fn()],
}));
const mockGlobalTime = { minTime: 1, maxTime: 2, selectedTime: '5m' };
jest.mock('react-redux', () => ({
useSelector: (selector: (state: unknown) => unknown): unknown =>
selector({
globalTime: { minTime: 1, maxTime: 2, selectedTime: '5m' },
}),
selector({ globalTime: mockGlobalTime }),
}));
jest.mock('../../DashboardSettings/Variables/variableAdapters', () => ({
@@ -150,3 +150,57 @@ describe('useVariableSelection — setSelection', () => {
expect(svcCycleId()).toBe(before + 1);
});
});
describe('useVariableSelection — what a time-range change enqueues', () => {
// Longer than FETCH_CYCLE_DEBOUNCE_MS, which the hook keeps private.
const PAST_DEBOUNCE = 400;
function reasons(): Record<string, string> {
return useDashboardStore.getState().variableCycleReasons;
}
beforeEach(() => {
jest.useFakeTimers();
mockGlobalTime.selectedTime = '5m';
useDashboardStore.setState({
variableValues: {},
variableFetchStates: {},
variableLastUpdated: {},
variableCycleIds: {},
variableCycleReasons: {},
variableResolvedEmpty: {},
variableFetchContext: null,
lastFetchAllKey: null,
});
});
afterEach(() => {
jest.useRealTimers();
});
// The tag is what stops the reconcile re-defaulting a user's selection.
it('tags every variable as a full cycle, overriding an earlier cascade tag', () => {
const { result, rerender } = renderHook(() =>
useVariableSelection(dashboard),
);
act(() => {
jest.advanceTimersByTime(PAST_DEBOUNCE);
});
expect(reasons()).toStrictEqual({ env: 'full-cycle', svc: 'full-cycle' });
// A value change re-scopes the dependent's options: it may drop what no longer applies.
act(() => {
result.current.setSelection('env', { value: ['prod'], allSelected: false });
});
expect(reasons().svc).toBe('value-cascade');
mockGlobalTime.selectedTime = '30m';
rerender();
act(() => {
jest.advanceTimersByTime(PAST_DEBOUNCE);
});
expect(reasons()).toStrictEqual({ env: 'full-cycle', svc: 'full-cycle' });
});
});

View File

@@ -6,6 +6,7 @@ import { DashboardDetailEvents } from 'pages/DashboardPageV2/constants/events';
import type { VariableSelection } from '../../selectionTypes';
import { areSelectionsEqual } from '../../utils/resolveVariableSelection';
import { selectionFromCommittedValues } from '../../utils/selectionUtils';
import OverflowValuesTooltip from './OverflowValuesTooltip';
import styles from '../../VariablesBar.module.scss';
@@ -75,13 +76,23 @@ function ValueSelector({
options.every((option) => draft.includes(option));
const commit = (values: string[]): void => {
// CustomMultiSelect emits the full value set when ALL is picked.
const isAll =
showAllOption &&
options.length > 0 &&
options.every((option) => values.includes(option));
const next: VariableSelection =
values.length === 0 ? emptyFallback : { value: values, allSelected: isAll };
// A close that left the list as it opened commits nothing — else a pick covering
// every option this window offers would be promoted to a standing ALL.
if (
areSelectionsEqual(
{ value: values, allSelected: false },
{ value: committedValues, allSelected: false },
)
) {
return;
}
const next = selectionFromCommittedValues({
values,
options,
showAllOption,
emptyFallback,
});
// Closing without actually changing the selection must not re-fire onChange —
// that would needlessly re-cascade to dependent variables/panels.

View File

@@ -1,6 +1,11 @@
import { useEffect } from 'react';
import type { VariableFormModel } from '../../DashboardSettings/Variables/variableFormModel';
import {
selectVariableCycleReason,
VariableCycleReason,
} from '../../store/slices/variableFetchSlice';
import { useDashboardStore } from '../../store/useDashboardStore';
import { reconcileWithOptions } from '../utils/resolveVariableSelection';
import type { VariableSelection } from '../selectionTypes';
@@ -9,6 +14,9 @@ import type { VariableSelection } from '../selectionTypes';
* `onAutoSelect` only when the value must change. The reconcile rule lives in
* {@link reconcileWithOptions} (shared with seed + payload defaulting) so the bar
* and the panel query can never disagree about a variable's default.
*
* Only a value cascade may re-default the selection; a full cycle (time range,
* reload) leaves the user's pick alone. Types with no cycle of their own reconcile.
*/
export function useAutoSelect(
variable: VariableFormModel,
@@ -16,8 +24,14 @@ export function useAutoSelect(
selection: VariableSelection,
onAutoSelect: (selection: VariableSelection) => void,
): void {
const cycleReason = useDashboardStore(
selectVariableCycleReason(variable.name),
);
useEffect(() => {
const next = reconcileWithOptions(variable, selection, options);
const next = reconcileWithOptions(variable, selection, options, {
preserveSelection: cycleReason === VariableCycleReason.FullCycle,
});
if (next) {
onAutoSelect(next);
}

View File

@@ -10,6 +10,11 @@ export interface VariableSelection {
value: SelectedVariableValue;
/** True when every option is selected ("ALL"); for dynamic vars value may be null. */
allSelected: boolean;
/**
* Entries of `value` the user typed rather than picked. Never in any option list,
* so the reconcile keeps them instead of reading them as invalid.
*/
customValues?: string[];
}
/** Selected values for a dashboard's variables, keyed by variable name. */

View File

@@ -134,12 +134,23 @@ export function resolveDefaultSelection(
return { value: model.multiSelect ? [] : '', allSelected: false };
}
interface ReconcileOptions {
/**
* Set when no other variable caused this refetch (time-range change, reload): the
* selection then outranks the options and is kept as-is. Leave false for a
* dependency cascade, where a selection that no longer applies must give way.
*/
preserveSelection?: boolean;
}
/**
* Reconciles a variable's current selection against its freshly-fetched options.
* Returns the next selection, or null when nothing should change (a valid pick is
* left untouched — local-first). Behaviour, in order:
* - materialize ALL to the full option set (query/custom);
* - keep a still-valid multi-select subset, dropping only invalid entries;
* - keep a multi-select selection outright when `preserveSelection` is set;
* - keep a still-valid multi-select subset, dropping only entries the list no longer
* offers and the user did not type in (`customValues`);
* - otherwise auto-pick the default (or first option) so dependent variables and
* panels always resolve against a usable value.
*/
@@ -147,6 +158,7 @@ export function reconcileWithOptions(
model: VariableFormModel,
current: VariableSelection,
options: string[],
{ preserveSelection = false }: ReconcileOptions = {},
): VariableSelection | null {
if (options.length === 0) {
return null;
@@ -161,13 +173,31 @@ export function reconcileWithOptions(
Array.isArray(current.value) &&
current.value.length > 0
) {
const valid = current.value.map(String).filter((c) => options.includes(c));
// A pick this window has no data for is still the user's filter; re-defaulting it
// here is what widened a single pick to ALL on every time-range change.
if (preserveSelection) {
return null;
}
// A typed value is in no option list, so it is never "no longer offered".
const custom = new Set(current.customValues ?? []);
const valid = current.value
.map(String)
.filter((c) => options.includes(c) || custom.has(c));
if (valid.length === current.value.length) {
return null;
}
return valid.length > 0
? { value: valid, allSelected: false }
: fillDefault(model, options);
if (valid.length === 0) {
return fillDefault(model, options);
}
const customValues = valid.filter((v) => custom.has(v));
return {
value: valid,
allSelected: false,
...(customValues.length > 0 && { customValues }),
};
}
if (!model.multiSelect) {

View File

@@ -47,6 +47,43 @@ export function hasUsableValue(
return value !== '' && value !== null && value !== undefined;
}
interface CommittedValues {
values: string[];
options: string[];
showAllOption: boolean;
emptyFallback: VariableSelection;
}
/**
* The selection a multi-select commit resolves to. Options are known only here, so
* this is where a value the list never offered is recorded as typed in.
*/
export function selectionFromCommittedValues({
values,
options,
showAllOption,
emptyFallback,
}: CommittedValues): VariableSelection {
if (values.length === 0) {
return emptyFallback;
}
const customValues = values.filter((value) => !options.includes(value));
// ALL re-materializes to the option set, so a set carrying a typed value is not ALL
// — the next refetch would expand it back and drop what the user typed.
const allSelected =
showAllOption &&
options.length > 0 &&
customValues.length === 0 &&
options.every((option) => values.includes(option));
return {
value: values,
allSelected,
...(customValues.length > 0 && { customValues }),
};
}
/** Flatten the selection map into the `{ name: value }` payload a query expects. */
export function selectionToPayload(
selection: VariableSelectionMap,

View File

@@ -34,6 +34,7 @@ function reset(names: string[], context: VariableFetchContext): void {
variableFetchStates: {},
variableLastUpdated: {},
variableCycleIds: {},
variableCycleReasons: {},
variableFetchContext: null,
});
store().initVariableFetch(names, context);
@@ -133,6 +134,33 @@ describe('variableFetchSlice', () => {
expect(states().q1).toBe('error');
expect(states().q2).toBe('idle');
});
// The reason is what tells the post-fetch reconcile whether it may re-default a
// selection: a full cycle must not, a value cascade must.
it('tags a full cycle, then re-tags only the cascaded variables', () => {
store().enqueueFetchAll();
expect(store().variableCycleReasons).toStrictEqual({
q1: 'full-cycle',
q2: 'full-cycle',
d1: 'full-cycle',
d2: 'full-cycle',
});
resolve('q1');
store().enqueueDescendants('q1');
expect(store().variableCycleReasons).toStrictEqual({
q1: 'full-cycle',
q2: 'value-cascade',
d1: 'full-cycle',
d2: 'full-cycle',
});
});
it('drops the reason for a variable that no longer exists', () => {
store().enqueueFetchAll();
store().initVariableFetch(['q1'], context);
expect(store().variableCycleReasons).toStrictEqual({ q1: 'full-cycle' });
});
});
describe('variableFetchSlice — query depends on a dynamic', () => {

View File

@@ -9,6 +9,7 @@ import {
type FetchMaps,
isVariableInActiveFetchState,
resolveFetchState,
VariableCycleReason,
VariableFetchState,
} from './variableFetchSlice.utils';
@@ -30,7 +31,10 @@ function queryParentsHaveValues(
);
}
export { VariableFetchState } from './variableFetchSlice.utils';
export {
VariableCycleReason,
VariableFetchState,
} from './variableFetchSlice.utils';
/**
* Runtime fetch orchestration for dashboard variables — native port of V1's
@@ -45,6 +49,8 @@ export interface VariableFetchSlice {
variableFetchStates: Record<string, VariableFetchState>;
variableLastUpdated: Record<string, number>;
variableCycleIds: Record<string, number>;
/** Why each variable's current cycle was enqueued, read by the post-fetch reconcile. */
variableCycleReasons: Record<string, VariableCycleReason>;
/**
* Whether a QUERY/DYNAMIC variable settled its fetch with zero options (so it
* will never get a value). Lets a dependent panel fall through to "no data"
@@ -106,6 +112,7 @@ export const createVariableFetchSlice: StateCreator<
variableFetchStates: {},
variableLastUpdated: {},
variableCycleIds: {},
variableCycleReasons: {},
variableResolvedEmpty: {},
variableFetchContext: null,
lastFetchAllKey: null,
@@ -115,6 +122,7 @@ export const createVariableFetchSlice: StateCreator<
variableFetchStates: {},
variableLastUpdated: {},
variableCycleIds: {},
variableCycleReasons: {},
variableResolvedEmpty: {},
variableFetchContext: null,
lastFetchAllKey: null,
@@ -132,6 +140,7 @@ export const createVariableFetchSlice: StateCreator<
initVariableFetch: (names, context): void => {
const maps = cloneMaps(get());
const resolvedEmpty = { ...get().variableResolvedEmpty };
const reasons = { ...get().variableCycleReasons };
names.forEach((name) => {
if (!maps.states[name]) {
maps.states[name] = VariableFetchState.Idle;
@@ -144,12 +153,14 @@ export const createVariableFetchSlice: StateCreator<
delete maps.lastUpdated[name];
delete maps.cycleIds[name];
delete resolvedEmpty[name];
delete reasons[name];
}
});
set({
variableFetchStates: maps.states,
variableLastUpdated: maps.lastUpdated,
variableCycleIds: maps.cycleIds,
variableCycleReasons: reasons,
variableResolvedEmpty: resolvedEmpty,
variableFetchContext: context,
});
@@ -171,6 +182,11 @@ export const createVariableFetchSlice: StateCreator<
dynamicVariableOrder,
} = variableFetchContext;
const maps = cloneMaps(get());
const reasons = { ...get().variableCycleReasons };
const bump = (name: string): void => {
maps.cycleIds[name] = (maps.cycleIds[name] || 0) + 1;
reasons[name] = VariableCycleReason.FullCycle;
};
// Query variables wait only for their QUERY parents. A DYNAMIC parent does not
// gate: its option fetch feeds only its own dropdown, while its selected value
@@ -178,7 +194,7 @@ export const createVariableFetchSlice: StateCreator<
// dependent query substitutes it immediately and refetches via the cascade if
// it later changes. Text/custom parents resolve synchronously, so nothing waits.
queryVariableOrder.forEach((name) => {
maps.cycleIds[name] = (maps.cycleIds[name] || 0) + 1;
bump(name);
const parents = dependencyData.parentGraph[name] || [];
const hasQueryParents = parents.some((p) => variableTypes[p] === 'QUERY');
maps.states[name] = hasQueryParents
@@ -192,7 +208,7 @@ export const createVariableFetchSlice: StateCreator<
const orderedQuery = new Set(queryVariableOrder);
Object.keys(variableTypes).forEach((name) => {
if (variableTypes[name] === 'QUERY' && !orderedQuery.has(name)) {
maps.cycleIds[name] = (maps.cycleIds[name] || 0) + 1;
bump(name);
maps.states[name] = resolveFetchState(maps, name);
}
});
@@ -203,7 +219,7 @@ export const createVariableFetchSlice: StateCreator<
// populate fast even when query variables are slow; a sibling selection change
// later refetches them via `enqueueDescendantsBatch`.
dynamicVariableOrder.forEach((name) => {
maps.cycleIds[name] = (maps.cycleIds[name] || 0) + 1;
bump(name);
maps.states[name] = resolveFetchState(maps, name);
});
@@ -211,6 +227,7 @@ export const createVariableFetchSlice: StateCreator<
variableFetchStates: maps.states,
variableLastUpdated: maps.lastUpdated,
variableCycleIds: maps.cycleIds,
variableCycleReasons: reasons,
lastFetchAllKey: key ?? get().lastFetchAllKey,
});
},
@@ -290,6 +307,11 @@ export const createVariableFetchSlice: StateCreator<
const { dependencyData, variableTypes, dynamicVariableOrder } =
variableFetchContext;
const maps = cloneMaps(get());
const reasons = { ...get().variableCycleReasons };
const bump = (name: string): void => {
maps.cycleIds[name] = (maps.cycleIds[name] || 0) + 1;
reasons[name] = VariableCycleReason.ValueCascade;
};
const changed = new Set(names);
// Callers commit values before this runs, so the gate sees the new parent values.
const selection = selectVariableValues(get().dashboardId)(get());
@@ -305,7 +327,7 @@ export const createVariableFetchSlice: StateCreator<
});
});
queryDescendants.forEach((desc) => {
maps.cycleIds[desc] = (maps.cycleIds[desc] || 0) + 1;
bump(desc);
maps.states[desc] = queryParentsHaveValues(
desc,
variableFetchContext,
@@ -322,7 +344,7 @@ export const createVariableFetchSlice: StateCreator<
dynamicVariableOrder
.filter((dynName) => !changed.has(dynName))
.forEach((dynName) => {
maps.cycleIds[dynName] = (maps.cycleIds[dynName] || 0) + 1;
bump(dynName);
maps.states[dynName] = resolveFetchState(maps, dynName);
});
}
@@ -331,6 +353,7 @@ export const createVariableFetchSlice: StateCreator<
variableFetchStates: maps.states,
variableLastUpdated: maps.lastUpdated,
variableCycleIds: maps.cycleIds,
variableCycleReasons: reasons,
});
},
});
@@ -347,6 +370,12 @@ export const selectVariableCycleId =
(state: DashboardStore): number =>
state.variableCycleIds[name] ?? 0;
/** Selector: why a variable's cycle was enqueued. Undefined for types that never fetch. */
export const selectVariableCycleReason =
(name: string) =>
(state: DashboardStore): VariableCycleReason | undefined =>
state.variableCycleReasons[name];
/** Selector: whether a variable has completed at least one fetch. */
export const selectVariableFetchedOnce =
(name: string) =>

View File

@@ -7,6 +7,14 @@ export enum VariableFetchState {
Error = 'error',
}
/** Why a cycle was started — only a cascade may re-default a user's selection. */
export enum VariableCycleReason {
/** `enqueueFetchAll`: load, time-range or variable-order change. */
FullCycle = 'full-cycle',
/** `enqueueDescendantsBatch`: a parent or sibling variable's value changed. */
ValueCascade = 'value-cascade',
}
/** Mutable clones a fetch action works over before committing back in one `set`. */
export interface FetchMaps {
states: Record<string, VariableFetchState>;

View File

@@ -239,16 +239,9 @@ func processJSONParser(parent *pipelinetypes.PipelineOperator) ([]pipelinetypes.
return nil, errors.NewInternalf(CodeInvalidOperatorType, "operator type received %s", parent.Type)
}
parseFromNotNilCheck, err := fieldNotNilCheck(parent.ParseFrom)
if err != nil {
return nil, errors.WrapInvalidInputf(err, CodeFieldNilCheckType,
"couldn't generate nil check for parseFrom of json parser op %s: %s", parent.Name, err,
)
}
parent.If = fmt.Sprintf(
`%s && ((type(%s) == "string" && isJSON(%s) && type(fromJSON(unquote(%s))) == "map" ) || type(%s) == "map")`,
parseFromNotNilCheck, parent.ParseFrom, parent.ParseFrom, parent.ParseFrom, parent.ParseFrom,
)
// on_error: send_quiet replaces the expensive isJSON `if` check;
// parse failures pass the record through unchanged without noisy logs.
parent.OnError = signozstanzahelper.SendOnErrorQuiet
if parent.EnableFlattening {
parent.MaxFlatteningDepth = constants.MaxJSONFlatteningDepth
}
@@ -298,7 +291,7 @@ func processJSONParser(parent *pipelinetypes.PipelineOperator) ([]pipelinetypes.
}
// JSONMapping: host
err = generateMoveOperators(mapping[pipelinetypes.Host], `resource["host.name"]`)
err := generateMoveOperators(mapping[pipelinetypes.Host], `resource["host.name"]`)
if err != nil {
return nil, err
}

View File

@@ -324,6 +324,17 @@ func TestNoCollectorErrorsFromProcessorsForMismatchedLogs(t *testing.T) {
makeTestLog("mismatching log", map[string]string{
"test_json": "bad json",
}),
}, {
"json parser should quietly ignore log with non JSON body",
pipelinetypes.PipelineOperator{
ID: "json",
Type: "json_parser",
Enabled: true,
Name: "json parser",
ParseFrom: "body",
ParseTo: "attributes",
},
makeTestLog("plain text log", map[string]string{}),
}, {
"move parser should ignore non matching logs",
pipelinetypes.PipelineOperator{
@@ -894,8 +905,8 @@ func TestProcessJSONParser_WithFlatteningAndMapping(t *testing.T) {
require.Equal(t, 1, parentOp.MaxFlatteningDepth)
require.Nil(t, parentOp.Mapping) // Mapping should be removed
require.Nil(t, parent.Mapping) // Mapping should be removed
require.Contains(t, parentOp.If, `isJSON(body)`)
require.Contains(t, parentOp.If, `type(body)`)
require.Empty(t, parentOp.If)
require.Equal(t, signozstanzahelper.SendOnErrorQuiet, parentOp.OnError)
require.Equal(t, 1+totalOps, len(ops))
@@ -951,7 +962,8 @@ func TestProcessJSONParser_WithoutMapping(t *testing.T) {
require.True(t, op.EnableFlattening)
require.True(t, op.EnablePaths)
require.Equal(t, "parsed", op.PathPrefix)
require.Contains(t, op.If, `isJSON(body)`)
require.Empty(t, op.If)
require.Equal(t, signozstanzahelper.SendOnErrorQuiet, op.OnError)
}
func TestProcessJSONParser_Simple(t *testing.T) {
@@ -975,7 +987,8 @@ func TestProcessJSONParser_Simple(t *testing.T) {
require.False(t, op.EnableFlattening)
require.False(t, op.EnablePaths)
require.Equal(t, "", op.PathPrefix)
require.Contains(t, op.If, `isJSON(body)`)
require.Empty(t, op.If)
require.Equal(t, signozstanzahelper.SendOnErrorQuiet, op.OnError)
}
func TestProcessJSONParser_InvalidType(t *testing.T) {

View File

@@ -56,17 +56,6 @@ func QueryStringToKeysSelectors(query string) []*telemetrytypes.FieldKeySelector
FieldDataType: key.FieldDataType,
})
}
// todo(tushar): consider reverting changes done to this method in below PR to avoid scope specific checks
// https://github.com/SigNoz/signoz/issues/11374
if key.FieldContext == telemetrytypes.FieldContextScope {
keys = append(keys, &telemetrytypes.FieldKeySelector{
Name: key.FieldContext.StringValue() + "." + key.Name,
Signal: key.Signal,
FieldContext: telemetrytypes.FieldContextUnspecified, // this allows 'scope.' prefix for keys with other context as well
FieldDataType: key.FieldDataType,
})
}
}
}

View File

@@ -72,23 +72,6 @@ func TestQueryToKeys(t *testing.T) {
},
},
},
{
query: `scope.version = '1.0.0'`,
expectedKeys: []telemetrytypes.FieldKeySelector{
{
Name: "version",
Signal: telemetrytypes.SignalUnspecified,
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
{
Name: "scope.version",
Signal: telemetrytypes.SignalUnspecified,
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
},
},
}
for _, testCase := range testCases {

View File

@@ -373,94 +373,6 @@ func TestStatementBuilder(t *testing.T) {
},
expectedErr: nil,
},
{
name: "scope.name filter and group by",
requestType: qbtypes.RequestTypeTimeSeries,
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.TraceAggregation{
{
Expression: "count()",
},
},
Filter: &qbtypes.Filter{
Expression: "scope.name = 'opentelemetry-io'",
},
Limit: 10,
GroupBy: []qbtypes.GroupByKey{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "scope.name",
FieldContext: telemetrytypes.FieldContextScope,
},
},
},
},
expected: qbtypes.Statement{
Query: "WITH __limit_cte AS (SELECT toString(multiIf(scope.name::String IS NOT NULL, scope.name::String, NULL)) AS `scope.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.name::String = ? AND scope.name::String IS NOT NULL) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `scope.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(scope.name::String IS NOT NULL, scope.name::String, NULL)) AS `scope.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.name::String = ? AND scope.name::String IS NOT NULL) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`scope.name`) GLOBAL IN (SELECT `scope.name` FROM __limit_cte) GROUP BY ts, `scope.name`",
Args: []any{"opentelemetry-io", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "opentelemetry-io", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
expectedErr: nil,
},
{
name: "scope.version filter with scope.name group by",
requestType: qbtypes.RequestTypeTimeSeries,
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.TraceAggregation{
{
Expression: "count()",
},
},
Filter: &qbtypes.Filter{
Expression: "scope.version = '1.0.0'",
},
Limit: 10,
GroupBy: []qbtypes.GroupByKey{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "scope.name",
FieldContext: telemetrytypes.FieldContextScope,
},
},
},
},
expected: qbtypes.Statement{
Query: "WITH __limit_cte AS (SELECT toString(multiIf(scope.name::String IS NOT NULL, scope.name::String, NULL)) AS `scope.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.version::String = ? AND scope.version::String IS NOT NULL) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `scope.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(scope.name::String IS NOT NULL, scope.name::String, NULL)) AS `scope.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.version::String = ? AND scope.version::String IS NOT NULL) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`scope.name`) GLOBAL IN (SELECT `scope.name` FROM __limit_cte) GROUP BY ts, `scope.name`",
Args: []any{"1.0.0", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1.0.0", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
expectedErr: nil,
},
{
name: "scope.version filter only (no scope field in group by)",
requestType: qbtypes.RequestTypeTimeSeries,
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.TraceAggregation{
{
Expression: "count()",
},
},
Filter: &qbtypes.Filter{
Expression: "scope.version = '1.0.0'",
},
Limit: 10,
GroupBy: []qbtypes.GroupByKey{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
},
},
},
},
expected: qbtypes.Statement{
Query: "WITH __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.version::String = ? AND scope.version::String IS NOT NULL) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.version::String = ? AND scope.version::String IS NOT NULL) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
Args: []any{"1.0.0", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1.0.0", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
},
}
fl := flaggertest.New(t)
@@ -887,52 +799,6 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
},
expectedErr: nil,
},
{
name: "List query with scope filter only (no scope in select or group by)",
requestType: qbtypes.RequestTypeRaw,
keysMap: map[string][]*telemetrytypes.TelemetryFieldKey{
"scope.version": {
{
Name: "scope.version",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
},
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Filter: &qbtypes.Filter{
Expression: "scope.version = '1.0.0'",
},
Limit: 10,
},
expected: qbtypes.Statement{
Query: "SELECT timestamp AS `timestamp`, trace_id AS `trace_id`, span_id AS `span_id`, trace_state AS `trace_state`, parent_span_id AS `parent_span_id`, flags AS `flags`, name AS `name`, kind AS `kind`, kind_string AS `kind_string`, duration_nano AS `duration_nano`, status_code AS `status_code`, status_message AS `status_message`, status_code_string AS `status_code_string`, events AS `events`, links AS `links`, response_status_code AS `response_status_code`, external_http_url AS `external_http_url`, http_url AS `http_url`, external_http_method AS `external_http_method`, http_method AS `http_method`, http_host AS `http_host`, db_name AS `db_name`, db_operation AS `db_operation`, has_error AS `has_error`, is_remote AS `is_remote`, attributes_string, attributes_number, attributes_bool, resources_string FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.version::String = ? AND scope.version::String IS NOT NULL) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"1.0.0", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10},
},
},
{
// Regression test: scope.version in selectFields with no metadata (isColumn=true filters it out)
// must still produce scope.version::String, not scope.attributes.version::String
name: "scope.version in selectFields only, no metadata (intrinsic field fallback)",
requestType: qbtypes.RequestTypeRaw,
keysMap: map[string][]*telemetrytypes.TelemetryFieldKey{},
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Filter: &qbtypes.Filter{},
SelectFields: []telemetrytypes.TelemetryFieldKey{
{Name: "scope.version", FieldContext: telemetrytypes.FieldContextUnspecified},
},
Limit: 10,
},
expected: qbtypes.Statement{
Query: "SELECT timestamp AS `timestamp`, trace_id AS `trace_id`, span_id AS `span_id`, scope.version::String AS `scope.version` FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10},
},
},
}
for _, c := range cases {

View File

@@ -180,7 +180,7 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
`CASE
// WHEN tagType = 'spanfield' THEN 1
WHEN tagType = 'resource' THEN 2
WHEN tagType = 'scope' THEN 3
// WHEN tagType = 'scope' THEN 3
WHEN tagType = 'tag' THEN 4
ELSE 5
END as priority`,

View File

@@ -121,20 +121,6 @@ var (
FieldContext: telemetrytypes.FieldContextSpan,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
"scope.name": {
Name: "scope.name",
Description: "Instrumentation scope name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
"scope.version": {
Name: "scope.version",
Description: "Instrumentation scope version",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
}
IntrinsicFieldsDeprecated = map[string]telemetrytypes.TelemetryFieldKey{
"traceID": {

View File

@@ -52,7 +52,6 @@ var (
ValueType: schema.ColumnTypeString,
}},
"resource": {Name: "resource", Type: schema.JSONColumnType{}},
"scope": {Name: "scope", Type: schema.JSONColumnType{}},
"events": {Name: "events", Type: schema.ArrayColumnType{
ElementType: schema.ColumnTypeString,
@@ -177,7 +176,7 @@ func (m *fieldMapper) getColumn(
case telemetrytypes.FieldContextResource:
return []*schema.Column{indexV3Columns["resource"], indexV3Columns["resources_string"]}, nil
case telemetrytypes.FieldContextScope:
return []*schema.Column{indexV3Columns["scope"]}, nil
return []*schema.Column{}, qbtypes.ErrColumnNotFound
case telemetrytypes.FieldContextAttribute:
switch key.FieldDataType {
case telemetrytypes.FieldDataTypeString:
@@ -288,24 +287,14 @@ func (m *fieldMapper) resolveColumnExprs(
switch column.Type.GetType() {
case schema.ColumnTypeEnumJSON:
// json is only supported for resource context as of now
if key.FieldContext != telemetrytypes.FieldContextResource {
return nil, nil, nil, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns, got %s", key.FieldContext.String)
}
// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
// once clickHouse dependency is updated, we need to check if we can remove it.
switch key.FieldContext {
case telemetrytypes.FieldContextResource:
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
existExprs = append(existExprs, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
case telemetrytypes.FieldContextScope:
switch key.Name {
case "scope.name", "scope.version":
exprs = append(exprs, fmt.Sprintf("%s::String", key.Name))
existExprs = append(existExprs, fmt.Sprintf("%s IS NOT NULL", key.Name))
default:
exprs = append(exprs, fmt.Sprintf("%s.attributes.`%s`::String", columnName, key.Name))
existExprs = append(existExprs, fmt.Sprintf("%s.attributes.`%s` IS NOT NULL", columnName, key.Name))
}
default:
return nil, nil, nil, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource and scope context fields are supported for json columns, got %s", key.FieldContext.String)
}
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
existExprs = append(existExprs, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
case schema.ColumnTypeEnumString,
schema.ColumnTypeEnumUInt64,
schema.ColumnTypeEnumUInt32,

View File

@@ -83,33 +83,6 @@ func TestGetFieldKeyName(t *testing.T) {
expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`, `resource_string_deployment$$environment`, NULL)",
expectedError: nil,
},
{
name: "Scope field - scope.name",
key: telemetrytypes.TelemetryFieldKey{
Name: "scope.name",
FieldContext: telemetrytypes.FieldContextScope,
},
expectedResult: "scope.name::String",
expectedError: nil,
},
{
name: "Scope field - scope.version",
key: telemetrytypes.TelemetryFieldKey{
Name: "scope.version",
FieldContext: telemetrytypes.FieldContextScope,
},
expectedResult: "scope.version::String",
expectedError: nil,
},
{
name: "Scope field - custom attribute",
key: telemetrytypes.TelemetryFieldKey{
Name: "custom.attr",
FieldContext: telemetrytypes.FieldContextScope,
},
expectedResult: "scope.attributes.`custom.attr`::String",
expectedError: nil,
},
{
// Query like `attribute.attribute_string:string` should resolve to `attributes_string['attribute_string']`.
name: "Attribute key whose name collides with contextual map column resolves as a map lookup",

View File

@@ -113,20 +113,6 @@ func BuildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytype
FieldDataType: telemetrytypes.FieldDataTypeBool,
},
},
"scope.name": {
{
Name: "scope.name",
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
"scope.version": {
{
Name: "scope.version",
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
}
for _, keys := range keysMap {
for _, key := range keys {

View File

@@ -995,8 +995,6 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"cloud.provider": "integration",
"cloud.account.id": "000",
"trace_id": "corrupt_data",
"scope_name": "corrupt_data",
"scope.scope.name": "corrupt_data",
},
attributes={
"net.transport": "IP.TCP",
@@ -1005,10 +1003,7 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"http.request.method": "POST",
"http.response.status_code": "200",
"timestamp": "corrupt_data",
"version": "1.0.0",
"scope.scope.version": "1.0.0",
},
scope={"name": "io.signoz.http.server", "version": "2.0.0"},
),
Traces(
timestamp=now - timedelta(seconds=3.5),
@@ -1028,24 +1023,12 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"cloud.provider": "integration",
"cloud.account.id": "000",
"timestamp": "corrupt_data",
"scope.attributes.name": "corrupt_data",
},
attributes={
"db.name": "integration",
"db.operation": "SELECT",
"db.statement": "SELECT * FROM integration",
"trace_d": "corrupt_data",
"scope.attributes.version": "corrupt_data",
},
scope={
"name": "io.opentelemetry.contrib.http",
"version": "1.0.0",
"attributes": {
"telemetry.sdk.language": "cpp",
"name": "not-the-real-name",
"version": "not-the-real-version",
"attributes": "literally-a-key-named-attributes",
},
},
),
Traces(
@@ -1066,15 +1049,12 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"cloud.provider": "integration",
"cloud.account.id": "000",
"duration_nano": "corrupt_data",
"scope.scope.attributes.version": "corrupt_data",
},
attributes={
"http.request.method": "PATCH",
"http.status_code": "404",
"id": "1",
"scope.scope.version": "corrupt_data",
},
scope={"name": "io.signoz.http.client", "version": "2.0.0"},
),
Traces(
timestamp=now - timedelta(seconds=1),
@@ -1093,7 +1073,6 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"host.name": "linux-001",
"cloud.provider": "integration",
"cloud.account.id": "001",
"scope.scope.version": "corrupt_data",
},
attributes={
"message.type": "SENT",
@@ -1101,10 +1080,7 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"messaging.message.id": "001",
"duration_nano": "corrupt_data",
"id": 1,
"scope": "corrupt_data",
"scope.attributes.name": "corrupt_data",
},
scope={"name": "io.signoz.messaging", "version": "3.0.0"},
),
]

View File

@@ -302,7 +302,6 @@ class Traces(ABC):
db_operation: str
has_error: bool
is_remote: str
scope_json: dict[str, Any]
resource: list[TracesResource]
tag_attributes: list[TracesTagAttributes]
@@ -328,7 +327,6 @@ class Traces(ABC):
links: list[TracesLink] = [],
trace_state: str = "",
flags: np.uint32 = 0,
scope: dict[str, Any] = {},
resource_write_mode: Literal["legacy_only", "dual_write"] = "dual_write",
) -> None:
if timestamp is None:
@@ -410,33 +408,6 @@ class Traces(ABC):
# Calculate resource fingerprint
self.resource_fingerprint = LogsOrTracesFingerprint(self.resources_string).calculate()
# Process scope mirroring the InstrumentationScope on the OTLP span.
scope_name = scope.get("name", "")
scope_version = scope.get("version", "")
scope_string = {k: str(v) for k, v in scope.get("attributes", {}).items()}
self.scope_json = {
"name": scope_name,
"version": scope_version,
"attributes": scope_string,
}
scope_keys = {"scope.name": scope_name, "scope.version": scope_version}
scope_keys.update(scope_string)
for k, v in scope_keys.items():
if v == "":
continue
self.tag_attributes.append(
TracesTagAttributes(
timestamp=timestamp,
tag_key=k,
tag_type="scope",
tag_data_type="string",
string_value=v,
number_value=None,
)
)
self.attribute_keys.append(TracesResourceOrAttributeKeys(name=k, datatype="string", tag_type="scope"))
# Process attributes by type and populate custom fields
self.attribute_string = {}
self.attributes_number = {}
@@ -689,7 +660,6 @@ class Traces(ABC):
self.has_error,
self.is_remote,
self.resource_json,
self.scope_json,
],
dtype=object,
)
@@ -721,7 +691,6 @@ class Traces(ABC):
attributes=data.get("attributes", {}),
trace_state=data.get("trace_state", ""),
flags=data.get("flags", 0),
scope=data.get("scope", {}),
)
@classmethod
@@ -861,7 +830,6 @@ def insert_traces_to_clickhouse(conn, traces: list[Traces]) -> None:
"has_error",
"is_remote",
"resource",
"scope",
],
data=[trace.np_arr() for trace in traces],
)

View File

@@ -359,15 +359,28 @@ def test_preview_logs_pipelines_success(
) -> None:
"""
Setup:
Create a preview request with a pipeline and sample logs.
Preview a json_parser pipeline with one JSON log and one plain-text log.
Tests:
1. Send preview request with valid pipeline configuration
2. Verify the preview processes logs correctly
3. Verify the response contains processed logs
1. JSON body gets parsed into attributes
2. Non-JSON body passes through unchanged instead of being dropped
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
empty_log_fields = {
"id": "",
"trace_id": "",
"span_id": "",
"trace_flags": 0,
"severity_text": "",
"severity_number": 0,
"attributes_string": {},
"attributes_int": {},
"attributes_float": {},
"attributes_bool": {},
"resources_string": {},
}
preview_payload = {
"pipelines": [
{
@@ -396,29 +409,25 @@ def test_preview_logs_pipelines_success(
{
"type": "json_parser",
"id": "json-parser-preview",
"orderId": 1,
"enabled": True,
"parse_from": "body",
"parse_to": "attributes",
"on_error": "send",
}
],
}
],
"logs": [
{
"body": '{"level": "info", "message": "Test log message", "timestamp": "2024-01-01T00:00:00Z"}',
"body": '{"level": "info", "message": "json log"}',
"timestamp": 1704067200000000000, # nanoseconds, not milliseconds
"id": "",
"trace_id": "",
"span_id": "",
"trace_flags": 0,
"severity_text": "",
"severity_number": 0,
"attributes_string": {},
"attributes_int": {},
"attributes_float": {},
"attributes_bool": {},
"resources_string": {"service.name": "test-service"},
}
**empty_log_fields,
},
{
"body": "plain text log that is not json",
"timestamp": 1704067201000000000,
**empty_log_fields,
},
],
}
@@ -435,13 +444,16 @@ def test_preview_logs_pipelines_success(
assert response.status_code == HTTPStatus.OK
response_data = response.json()
assert response_data["status"] == "success"
assert "data" in response_data
assert "logs" in response_data["data"]
assert len(response_data["data"]["logs"]) == 1
logs = response_data["data"]["logs"]
assert len(logs) == 2
# Verify the log was processed
processed_log = response_data["data"]["logs"][0]
assert "attributes_string" in processed_log or "attributes" in processed_log
json_log = next(log for log in logs if log["body"].startswith("{"))
assert json_log["attributes_string"]["level"] == "info"
assert json_log["attributes_string"]["message"] == "json log"
plain_log = next(log for log in logs if not log["body"].startswith("{"))
assert plain_log["body"] == "plain text log that is not json"
assert plain_log["attributes_string"] == {}
def test_create_multiple_pipelines_success(

View File

@@ -1199,13 +1199,6 @@ def test_traces_list_span_scope(
lambda x: {"duration_nano": int(x[1].duration_nano), "span_id": x[1].span_id, "timestamp": format_timestamp(x[1].timestamp), "trace_id": x[1].trace_id},
id="select_attribute_duration_order_intrinsic",
),
# Case 9: filter on the intrinsic scope.version. Only x[1] should match.
pytest.param(
BuilderQuery(signal="traces", name="A", select_fields=[TelemetryFieldKey("timestamp")], filter_expression="scope.version = '1.0.0'", limit=1),
HTTPStatus.OK,
lambda x: {"span_id": x[1].span_id, "timestamp": format_timestamp(x[1].timestamp), "trace_id": x[1].trace_id},
id="filter_scope_version",
),
],
)
def test_traces_list_with_corrupt_data(
@@ -1249,156 +1242,6 @@ def test_traces_list_with_corrupt_data(
assert get_rows(response)[0]["data"] == expected(traces)
@pytest.mark.parametrize(
"filter_expression,expected_indices",
[
# Intrinsic scope.name / scope.version resolve to the JSON sub-columns.
pytest.param("scope.name = 'io.signoz.payment'", [1], id="intrinsic_scope_name"),
pytest.param("scope.version = '2.3.1'", [0], id="intrinsic_scope_version"),
# A scope attribute resolves against the scope JSON column's attributes.
pytest.param("scope.telemetry.sdk.language = 'python'", [1], id="scope_attribute"),
# `env.tier` is a span attribute on span 0 and a scope attribute on
# span 1. Unprefixed -> no explicit context, so it is checked in every
# applicable context (attribute OR scope) and both spans match.
pytest.param("env.tier = 'gold'", [0, 1], id="bare_cross_context"),
# The explicit `scope.` prefix forces scope context only, so span 0's
# span attribute is ignored — only span 1 matches.
pytest.param("scope.env.tier = 'gold'", [1], id="scope_prefixed_cross_context"),
# `scope.name` matches BOTH the intrinsic scope.name field (span 0) and a
# scope attribute literally named `name` (span 1's scope attribute
# name='io.signoz.checkout').
pytest.param("scope.name = 'io.signoz.checkout'", [0, 1], id="scope_name_collision"),
# `scope.name` also matches a span attribute literally named `scope.name`
# (attribute context) — span 2 carries attribute scope.name='attr-scope-name'.
pytest.param("scope.name = 'attr-scope-name'", [2], id="scope_name_attribute_collision"),
# An unprefixed `name` resolves to the intrinsic span `name` column and a
# `name` scope attribute, but NOT the scope.name field. Span 2's span
# name and span 1's scope attribute `name` both equal 'io.signoz.checkout';
# span 0's scope.name field equals it too but is NOT matched.
pytest.param("name = 'io.signoz.checkout'", [1, 2], id="bare_name_excludes_scope_name_field"),
# A value that no resolvable key holds (scope.name/scope.version field,
# a `name`/`version` scope attribute, or a same-named attribute/resource)
# returns nothing.
pytest.param("scope.version = 'corrupt_data'", [], id="scope_version_no_match"),
pytest.param("scope.name = 'corrupt_data'", [], id="scope_name_no_match"),
],
)
def test_traces_list_with_scope_filter(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
filter_expression: str,
expected_indices: list[int],
) -> None:
"""
Setup three spans with different scope key resolution:
- x[0]: scope.name/version 'io.signoz.checkout'/'2.3.1'; span attribute
env.tier='gold'.
- x[1]: scope.name/version 'io.signoz.payment'/'4.5.6'; scope attributes
telemetry.sdk.language='python', env.tier='gold', and a `name` scope
attribute colliding with x[0]'s scope.name value.
- x[2]: span name 'io.signoz.checkout' (colliding with x[0]'s scope.name
value) and a span attribute literally named `scope.name`.
Tests:
- Filtering on scope.name / scope.version / a scope attribute.
- An unprefixed key is resolved across contexts (scope checked alongside
attribute / intrinsic), while a `scope.`-prefixed key is scope-only.
- `scope.name` hits the intrinsic field, a `name` scope attribute, and a
span attribute `scope.name` (cross-context), while a bare `name` hits
the span name column (and a `name` scope attribute) but never the
scope.name field.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
trace_id = TraceIdGenerator.trace_id()
span_ids = [TraceIdGenerator.span_id() for _ in range(3)]
traces = [
Traces(
timestamp=now - timedelta(seconds=4),
duration=timedelta(seconds=2),
trace_id=trace_id,
span_id=span_ids[0],
parent_span_id="",
name="GET /checkout",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
resources={"service.name": "checkout"},
attributes={"http.request.method": "GET", "env.tier": "gold"},
scope={
"name": "io.signoz.checkout",
"version": "2.3.1",
"attributes": {"telemetry.sdk.language": "go"},
},
),
Traces(
timestamp=now - timedelta(seconds=2),
duration=timedelta(seconds=1),
trace_id=trace_id,
span_id=span_ids[1],
parent_span_id="",
name="POST /pay",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
resources={"service.name": "payment"},
attributes={"http.request.method": "POST"},
# env.tier is a scope attribute here (cross-context with span 0);
# `name` is a scope attribute colliding with span 0's scope.name.
scope={
"name": "io.signoz.payment",
"version": "4.5.6",
"attributes": {
"telemetry.sdk.language": "python",
"env.tier": "gold",
"name": "io.signoz.checkout",
},
},
),
Traces(
timestamp=now - timedelta(seconds=1),
duration=timedelta(seconds=1),
trace_id=trace_id,
span_id=span_ids[2],
parent_span_id="",
# span name collides with span 0's scope.name value
name="io.signoz.checkout",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
resources={"service.name": "probe"},
# a span attribute named `scope.name`
attributes={"scope.name": "attr-scope-name"},
scope={"name": "span-gamma", "version": "9.9.9"},
),
]
insert_traces(traces)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
start_ms, end_ms = _query_window(now)
response = make_query_request(
signoz,
token,
start_ms=start_ms,
end_ms=end_ms,
request_type=RequestType.RAW,
queries=[
BuilderQuery(
signal="traces",
name="A",
select_fields=[TelemetryFieldKey("timestamp")],
filter_expression=filter_expression,
limit=10,
).to_dict()
],
)
assert response.status_code == HTTPStatus.OK, response.text
got_span_ids = {row["data"]["span_id"] for row in get_rows(response)}
expected_span_ids = {traces[i].span_id for i in expected_indices}
assert got_span_ids == expected_span_ids
@pytest.mark.parametrize("surface", ["filter", "select", "order"])
def test_traces_list_unknown_span_context_synthesizes(
signoz: types.SigNoz,