Compare commits

...

15 Commits

Author SHA1 Message Date
Nikhil Soni
800e49dd69 chore: remove un related changes to keep diff minimum 2026-06-02 16:58:02 +05:30
Nikhil Soni
80b2520e44 chore: move flamegraph after aggregation to reduce diff 2026-06-02 16:51:46 +05:30
Nikhil Soni
055a6cbd9b chore: update openapi specs 2026-06-02 16:36:12 +05:30
Nikhil Soni
099510a497 chore: extract out flamegraph building logic for easier review 2026-06-02 16:30:33 +05:30
Nikhil Soni
f3e6534d35 chore: ignore nil assigment lint error 2026-06-02 16:30:33 +05:30
Nikhil Soni
09fe9fa7cb chore: move exported methods to the top 2026-06-02 16:30:31 +05:30
Nikhil Soni
8b64247efa feat: query full spans for smaller traces 2026-06-02 16:23:42 +05:30
Nikhil Soni
7948b3091f feat: add api and module for flamegraph v3 2026-06-02 16:21:10 +05:30
Nikhil Soni
9bef79f664 feat: add method to enrich selected spans 2026-06-02 16:11:16 +05:30
Nikhil Soni
aea4a011f9 feat: add config for flamegraph 2026-06-02 16:11:16 +05:30
Nikhil Soni
b07335d969 chore: remove limit from request payload
It's a new api so doesn't need to be backward compatible
2026-06-02 16:11:16 +05:30
Nikhil Soni
c5f002110a feat: add types for flamegraph v3 in module structure 2026-06-02 16:11:16 +05:30
Vikrant Gupta
0963ff08cd feat(web): disable all integrations by default (#11539) 2026-06-02 09:32:20 +00:00
Nikhil Soni
e43aeb8e24 fix: add references in waterfall response (#11536)
* fix: add references in waterfall response

* chore: update openapi specs

* chore: mark reference a non nullable field
2026-06-02 08:42:26 +00:00
Aditya Singh
9074208b09 feat: added trace events (#11538) 2026-06-02 08:41:39 +00:00
22 changed files with 1033 additions and 18 deletions

View File

@@ -64,16 +64,16 @@ web:
settings:
posthog:
# Whether to enable PostHog in web.
enabled: true
enabled: false
appcues:
# Whether to enable Appcues in web.
enabled: true
enabled: false
sentry:
# Whether to enable Sentry in web.
enabled: true
enabled: false
pylon:
# Whether to enable Pylon in web.
enabled: true
enabled: false
##################### Cache #####################
cache:
@@ -440,6 +440,17 @@ tracedetail:
max_depth_to_auto_expand: 5
# Threshold below which all spans are returned without windowing.
max_limit_to_select_all_spans: 10000
flamegraph:
# Maximum number of BFS depth levels included in a windowed response.
max_selected_levels: 50
# Maximum spans per level before sampling is applied.
max_spans_per_level: 100
# Number of highest-latency spans always included when sampling a level.
sampling_top_latency_count: 5
# Number of timestamp buckets used for uniform sampling within a level.
sampling_bucket_count: 50
# Threshold below which all spans are returned without windowing or sampling.
select_all_spans_limit: 100000
##################### Authz #################################
authz:

View File

@@ -6516,6 +6516,58 @@ components:
- attribute
- resource
type: string
SpantypesFlamegraphSpan:
properties:
attributes:
additionalProperties: {}
type: object
durationNano:
minimum: 0
type: integer
event:
items:
$ref: '#/components/schemas/SpantypesEvent'
nullable: true
type: array
hasError:
type: boolean
level:
format: int64
type: integer
name:
type: string
parentSpanId:
type: string
resource:
additionalProperties:
type: string
type: object
serviceName:
type: string
spanId:
type: string
timestamp:
minimum: 0
type: integer
type: object
SpantypesGettableFlamegraphTrace:
properties:
endTimestampMillis:
format: int64
type: integer
hasMore:
type: boolean
spans:
items:
items:
$ref: '#/components/schemas/SpantypesFlamegraphSpan'
type: array
nullable: true
type: array
startTimestampMillis:
format: int64
type: integer
type: object
SpantypesGettableSpanMapperGroups:
properties:
items:
@@ -6572,6 +6624,24 @@ components:
nullable: true
type: array
type: object
SpantypesOtelSpanRef:
properties:
refType:
type: string
spanId:
type: string
traceId:
type: string
type: object
SpantypesPostableFlamegraph:
properties:
selectFields:
items:
$ref: '#/components/schemas/TelemetrytypesTelemetryFieldKey'
type: array
selectedSpanId:
type: string
type: object
SpantypesPostableSpanMapper:
properties:
config:
@@ -6835,6 +6905,10 @@ components:
type: string
parent_span_id:
type: string
references:
items:
$ref: '#/components/schemas/SpantypesOtelSpanRef'
type: array
resource:
additionalProperties:
type: string
@@ -6860,6 +6934,8 @@ components:
type: string
trace_state:
type: string
required:
- references
type: object
TagtypesPostableTag:
properties:
@@ -20012,6 +20088,75 @@ paths:
summary: Put profile in Zeus for a deployment.
tags:
- zeus
/api/v3/traces/{traceID}/flamegraph:
post:
deprecated: false
description: Returns the flamegraph view of spans for a given trace ID.
operationId: GetFlamegraph
parameters:
- in: path
name: traceID
required: true
schema:
type: string
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/SpantypesPostableFlamegraph'
responses:
"200":
content:
application/json:
schema:
properties:
data:
$ref: '#/components/schemas/SpantypesGettableFlamegraphTrace'
status:
type: string
required:
- status
- data
type: object
description: OK
"400":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Bad Request
"401":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Unauthorized
"403":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
"404":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Not Found
"500":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Internal Server Error
security:
- api_key:
- VIEWER
- tokenizer:
- VIEWER
summary: Get flamegraph view for a trace
tags:
- tracedetail
/api/v3/traces/{traceID}/waterfall:
post:
deprecated: false

View File

@@ -7675,6 +7675,81 @@ export enum SpantypesFieldContextDTO {
attribute = 'attribute',
resource = 'resource',
}
export type SpantypesFlamegraphSpanDTOAttributes = { [key: string]: unknown };
export type SpantypesFlamegraphSpanDTOResource = { [key: string]: string };
export interface SpantypesFlamegraphSpanDTO {
/**
* @type object
*/
attributes?: SpantypesFlamegraphSpanDTOAttributes;
/**
* @type integer
* @minimum 0
*/
durationNano?: number;
/**
* @type array,null
*/
event?: SpantypesEventDTO[] | null;
/**
* @type boolean
*/
hasError?: boolean;
/**
* @type integer
* @format int64
*/
level?: number;
/**
* @type string
*/
name?: string;
/**
* @type string
*/
parentSpanId?: string;
/**
* @type object
*/
resource?: SpantypesFlamegraphSpanDTOResource;
/**
* @type string
*/
serviceName?: string;
/**
* @type string
*/
spanId?: string;
/**
* @type integer
* @minimum 0
*/
timestamp?: number;
}
export interface SpantypesGettableFlamegraphTraceDTO {
/**
* @type integer
* @format int64
*/
endTimestampMillis?: number;
/**
* @type boolean
*/
hasMore?: boolean;
/**
* @type array,null
*/
spans?: SpantypesFlamegraphSpanDTO[][] | null;
/**
* @type integer
* @format int64
*/
startTimestampMillis?: number;
}
export type SpantypesSpanMapperGroupConditionDTOAnyOf = {
/**
* @type array,null
@@ -7768,6 +7843,21 @@ export interface SpantypesGettableTraceAggregationsDTO {
aggregations: SpantypesSpanAggregationResultDTO[];
}
export interface SpantypesOtelSpanRefDTO {
/**
* @type string
*/
refType?: string;
/**
* @type string
*/
spanId?: string;
/**
* @type string
*/
traceId?: string;
}
export type SpantypesWaterfallSpanDTOAttributesAnyOf = {
[key: string]: unknown;
};
@@ -7862,6 +7952,10 @@ export interface SpantypesWaterfallSpanDTO {
* @type string
*/
parent_span_id?: string;
/**
* @type array
*/
references: SpantypesOtelSpanRefDTO[];
/**
* @type object,null
*/
@@ -7957,6 +8051,17 @@ export interface SpantypesGettableWaterfallTraceDTO {
uncollapsedSpans?: string[] | null;
}
export interface SpantypesPostableFlamegraphDTO {
/**
* @type array
*/
selectFields?: TelemetrytypesTelemetryFieldKeyDTO[];
/**
* @type string
*/
selectedSpanId?: string;
}
export enum SpantypesSpanMapperOperationDTO {
move = 'move',
copy = 'copy',
@@ -10258,6 +10363,17 @@ export type GetHosts200 = {
status: string;
};
export type GetFlamegraphPathParameters = {
traceID: string;
};
export type GetFlamegraph200 = {
data: SpantypesGettableFlamegraphTraceDTO;
/**
* @type string
*/
status: string;
};
export type GetWaterfallPathParameters = {
traceID: string;
};

View File

@@ -12,6 +12,8 @@ import type {
} from 'react-query';
import type {
GetFlamegraph200,
GetFlamegraphPathParameters,
GetTraceAggregations200,
GetTraceAggregationsPathParameters,
GetWaterfall200,
@@ -19,6 +21,7 @@ import type {
GetWaterfallV4200,
GetWaterfallV4PathParameters,
RenderErrorResponseDTO,
SpantypesPostableFlamegraphDTO,
SpantypesPostableTraceAggregationsDTO,
SpantypesPostableWaterfallDTO,
} from '../sigNoz.schemas';
@@ -126,6 +129,105 @@ export const useGetTraceAggregations = <
> => {
return useMutation(getGetTraceAggregationsMutationOptions(options));
};
/**
* Returns the flamegraph view of spans for a given trace ID.
* @summary Get flamegraph view for a trace
*/
export const getFlamegraph = (
{ traceID }: GetFlamegraphPathParameters,
spantypesPostableFlamegraphDTO?: BodyType<SpantypesPostableFlamegraphDTO>,
signal?: AbortSignal,
) => {
return GeneratedAPIInstance<GetFlamegraph200>({
url: `/api/v3/traces/${traceID}/flamegraph`,
method: 'POST',
headers: { 'Content-Type': 'application/json' },
data: spantypesPostableFlamegraphDTO,
signal,
});
};
export const getGetFlamegraphMutationOptions = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown,
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof getFlamegraph>>,
TError,
{
pathParams: GetFlamegraphPathParameters;
data?: BodyType<SpantypesPostableFlamegraphDTO>;
},
TContext
>;
}): UseMutationOptions<
Awaited<ReturnType<typeof getFlamegraph>>,
TError,
{
pathParams: GetFlamegraphPathParameters;
data?: BodyType<SpantypesPostableFlamegraphDTO>;
},
TContext
> => {
const mutationKey = ['getFlamegraph'];
const { mutation: mutationOptions } = options
? options.mutation &&
'mutationKey' in options.mutation &&
options.mutation.mutationKey
? options
: { ...options, mutation: { ...options.mutation, mutationKey } }
: { mutation: { mutationKey } };
const mutationFn: MutationFunction<
Awaited<ReturnType<typeof getFlamegraph>>,
{
pathParams: GetFlamegraphPathParameters;
data?: BodyType<SpantypesPostableFlamegraphDTO>;
}
> = (props) => {
const { pathParams, data } = props ?? {};
return getFlamegraph(pathParams, data);
};
return { mutationFn, ...mutationOptions };
};
export type GetFlamegraphMutationResult = NonNullable<
Awaited<ReturnType<typeof getFlamegraph>>
>;
export type GetFlamegraphMutationBody =
| BodyType<SpantypesPostableFlamegraphDTO>
| undefined;
export type GetFlamegraphMutationError = ErrorType<RenderErrorResponseDTO>;
/**
* @summary Get flamegraph view for a trace
*/
export const useGetFlamegraph = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown,
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof getFlamegraph>>,
TError,
{
pathParams: GetFlamegraphPathParameters;
data?: BodyType<SpantypesPostableFlamegraphDTO>;
},
TContext
>;
}): UseMutationResult<
Awaited<ReturnType<typeof getFlamegraph>>,
TError,
{
pathParams: GetFlamegraphPathParameters;
data?: BodyType<SpantypesPostableFlamegraphDTO>;
},
TContext
> => {
return useMutation(getGetFlamegraphMutationOptions(options));
};
/**
* Returns the waterfall view of spans for a given trace ID with tree structure, metadata, and windowed pagination
* @summary Get waterfall view for a trace

View File

@@ -22,6 +22,7 @@ import styles from './AnalyticsPanel.module.scss';
interface AnalyticsPanelProps {
isOpen: boolean;
onClose: () => void;
onTabChange: (tab: string) => void;
}
const PANEL_WIDTH = 350;
@@ -32,6 +33,7 @@ const PANEL_MARGIN_BOTTOM = 50;
function AnalyticsPanel({
isOpen,
onClose,
onTabChange,
}: AnalyticsPanelProps): JSX.Element | null {
const aggregations = useTraceStore((s) => s.aggregations);
const colorByFieldName = useTraceStore((s) => s.colorByField.name);
@@ -118,7 +120,7 @@ function AnalyticsPanel({
/>
<div className={styles.body}>
<TabsRoot defaultValue="exec-time">
<TabsRoot defaultValue="exec-time" onValueChange={onTabChange}>
<TabsList variant="secondary">
<TabsTrigger value="exec-time" variant="secondary">
% exec time

View File

@@ -31,7 +31,12 @@ import Events from 'container/SpanDetailsDrawer/Events/Events';
import SpanLogs from 'container/SpanDetailsDrawer/SpanLogs/SpanLogs';
import { useSpanContextLogs } from 'container/SpanDetailsDrawer/SpanLogs/useSpanContextLogs';
import dayjs from 'dayjs';
import {
TraceDetailEventKeys,
TraceDetailEvents,
} from 'pages/TraceDetailsV3/events';
import { useMigratePinnedAttributes } from 'pages/TraceDetailsV3/hooks/useMigratePinnedAttributes';
import { useTraceDetailLogEvent } from 'pages/TraceDetailsV3/hooks/useTraceDetailLogEvent';
import {
getSpanAttribute,
getSpanDisplayData,
@@ -86,6 +91,16 @@ function SpanDetailsContent({
}): JSX.Element {
const FIVE_MINUTES_IN_MS = 5 * 60 * 1000;
const spanAttributeActions = useSpanAttributeActions();
const logTraceEvent = useTraceDetailLogEvent('v3', selectedSpan.trace_id);
const handleTabChange = useCallback(
(tab: string): void => {
logTraceEvent(TraceDetailEvents.SpanPanelTabChanged, {
[TraceDetailEventKeys.Tab]: tab,
[TraceDetailEventKeys.SpanId]: selectedSpan.span_id,
});
},
[logTraceEvent, selectedSpan.span_id],
);
const percentile = useSpanPercentile(selectedSpan);
const linkedSpans = useLinkedSpans((selectedSpan as any).references);
@@ -376,7 +391,7 @@ function SpanDetailsContent({
<div className={styles.tabsSection}>
{/* Step 9: ContentTabs */}
<TabsRoot defaultValue="overview">
<TabsRoot defaultValue="overview" onValueChange={handleTabChange}>
<TabsList variant="secondary">
<TabsTrigger value="overview" variant="secondary">
<Bookmark size={14} /> Overview

View File

@@ -1,4 +1,4 @@
import { useCallback, useState } from 'react';
import { useCallback, useRef, useState } from 'react';
import { useParams } from 'react-router-dom';
import { Button } from '@signozhq/ui/button';
import {
@@ -29,6 +29,8 @@ import KeyValueLabel from 'periscope/components/KeyValueLabel';
import { TraceDetailV2URLProps } from 'types/api/trace/getTraceV2';
import { DataSource } from 'types/common/queryBuilder';
import { TraceDetailEventKeys, TraceDetailEvents } from '../events';
import { useTraceDetailLogEvent } from '../hooks/useTraceDetailLogEvent';
import { useTraceStore } from '../stores/traceStore';
import AnalyticsPanel from '../SpanDetailsPanel/AnalyticsPanel/AnalyticsPanel';
import Filters from '../TraceWaterfall/TraceWaterfallStates/Success/Filters/Filters';
@@ -90,11 +92,35 @@ function TraceDetailsHeader({
const previewFields = useTraceStore((s) => s.previewFields);
const setPreviewFields = useTraceStore((s) => s.setPreviewFields);
const logTraceEvent = useTraceDetailLogEvent('v3', traceID || '');
const pageLoadedAtRef = useRef(Date.now());
const handleSwitchToOldView = useCallback((): void => {
logTraceEvent(TraceDetailEvents.ViewSwitched, {
[TraceDetailEventKeys.From]: 'v3',
[TraceDetailEventKeys.To]: 'v2',
[TraceDetailEventKeys.DwellMs]: Date.now() - pageLoadedAtRef.current,
});
setLocalStorageKey(LOCALSTORAGE.TRACE_DETAILS_PREFER_OLD_VIEW, 'true');
const oldUrl = `/trace-old/${traceID}${window.location.search}`;
history.replace(oldUrl);
}, [traceID]);
}, [traceID, logTraceEvent]);
const handleToggleAnalytics = useCallback((): void => {
logTraceEvent(TraceDetailEvents.AnalyticsPanelToggled, {
[TraceDetailEventKeys.Open]: !isAnalyticsOpen,
});
setIsAnalyticsOpen((prev) => !prev);
}, [logTraceEvent, isAnalyticsOpen]);
const handleAnalyticsTabChange = useCallback(
(tab: string): void => {
logTraceEvent(TraceDetailEvents.AnalyticsTabChanged, {
[TraceDetailEventKeys.Tab]: tab,
});
},
[logTraceEvent],
);
const handlePreviousBtnClick = useCallback((): void => {
if (hasInAppHistory()) {
@@ -167,7 +193,7 @@ function TraceDetailsHeader({
size="icon"
color="secondary"
aria-label="Analytics"
onClick={(): void => setIsAnalyticsOpen((prev) => !prev)}
onClick={handleToggleAnalytics}
>
<ChartPie size={14} />
</Button>
@@ -245,6 +271,7 @@ function TraceDetailsHeader({
<AnalyticsPanel
isOpen={isAnalyticsOpen}
onClose={(): void => setIsAnalyticsOpen(false)}
onTabChange={handleAnalyticsTabChange}
/>
</div>
);

View File

@@ -0,0 +1,38 @@
export enum TraceDetailEvents {
DataLoaded = 'Trace Detail: Data loaded',
ViewSwitched = 'Trace Detail: View switched',
FlameGraphToggled = 'Trace Detail: Flame graph toggled',
WaterfallToggled = 'Trace Detail: Waterfall toggled',
AnalyticsPanelToggled = 'Trace Detail: Analytics panel toggled',
AnalyticsTabChanged = 'Trace Detail: Analytics tab changed',
SpanPanelTabChanged = 'Trace Detail: Span panel tab changed',
}
export enum TraceDetailEventKeys {
// Injected on every event by useTraceDetailLogEvent
View = 'view',
TraceId = 'traceId',
// Data loaded — trace shape
TotalSpansCount = 'totalSpansCount',
NumServices = 'numServices',
TraceDurationMs = 'traceDurationMs',
HadErrors = 'hadErrors',
FlamegraphSampled = 'flamegraphSampled',
// Data loaded — persisted settings
SpanPanelVariant = 'spanPanelVariant',
ColorByField = 'colorByField',
PreviewFieldsCount = 'previewFieldsCount',
EntryPreferOldView = 'entryPreferOldView',
// View switched
From = 'from',
To = 'to',
DwellMs = 'dwellMs',
// Toggles / tabs
Expanded = 'expanded',
Open = 'open',
Tab = 'tab',
// Span panel tab changed
SpanId = 'spanId',
}
export type TraceDetailView = 'v2' | 'v3';

View File

@@ -0,0 +1,88 @@
import { act, renderHook } from '@testing-library/react';
import { TraceDetailEvents } from '../../events';
import { useTraceDetailLogEvent } from '../useTraceDetailLogEvent';
const logEventMock = jest.fn();
jest.mock('api/common/logEvent', () => ({
__esModule: true,
default: (...args: unknown[]): void => logEventMock(...args),
}));
describe('useTraceDetailLogEvent', () => {
beforeEach(() => {
logEventMock.mockClear();
});
it('injects view and traceId on every event', () => {
const { result } = renderHook(() =>
useTraceDetailLogEvent('v3', 'trace-123'),
);
act(() => {
result.current(TraceDetailEvents.DataLoaded, { totalSpansCount: 42 });
});
expect(logEventMock).toHaveBeenCalledTimes(1);
expect(logEventMock).toHaveBeenCalledWith(TraceDetailEvents.DataLoaded, {
view: 'v3',
traceId: 'trace-123',
totalSpansCount: 42,
});
});
it('injects view and traceId even when no attributes are passed', () => {
const { result } = renderHook(() =>
useTraceDetailLogEvent('v2', 'trace-456'),
);
act(() => {
result.current(TraceDetailEvents.ViewSwitched);
});
expect(logEventMock).toHaveBeenCalledWith(TraceDetailEvents.ViewSwitched, {
view: 'v2',
traceId: 'trace-456',
});
});
it('keeps a stable callback identity and emits the latest traceId', () => {
const { result, rerender } = renderHook(
({ traceId }) => useTraceDetailLogEvent('v3', traceId),
{ initialProps: { traceId: 'trace-1' } },
);
const firstIdentity = result.current;
rerender({ traceId: 'trace-2' });
expect(result.current).toBe(firstIdentity);
act(() => {
result.current(TraceDetailEvents.SpanPanelTabChanged, { spanId: 's1' });
});
expect(logEventMock).toHaveBeenCalledWith(
TraceDetailEvents.SpanPanelTabChanged,
{
view: 'v3',
traceId: 'trace-2',
spanId: 's1',
},
);
});
it('never throws if logEvent throws (analytics must not break the UI)', () => {
logEventMock.mockImplementationOnce(() => {
throw new Error('network down');
});
const { result } = renderHook(() =>
useTraceDetailLogEvent('v3', 'trace-123'),
);
expect(() => {
act(() => {
result.current(TraceDetailEvents.DataLoaded);
});
}).not.toThrow();
});
});

View File

@@ -0,0 +1,39 @@
import { useCallback, useRef } from 'react';
import logEvent from 'api/common/logEvent';
import {
TraceDetailEventKeys,
TraceDetailEvents,
TraceDetailView,
} from '../events';
export type TraceDetailLogEvent = (
event: TraceDetailEvents,
attributes?: Record<string, unknown>,
) => void;
export function useTraceDetailLogEvent(
view: TraceDetailView,
traceId: string,
): TraceDetailLogEvent {
const contextRef = useRef({ view, traceId });
contextRef.current = { view, traceId };
return useCallback(
(
event: TraceDetailEvents,
attributes: Record<string, unknown> = {},
): void => {
try {
void logEvent(event, {
[TraceDetailEventKeys.View]: contextRef.current.view,
[TraceDetailEventKeys.TraceId]: contextRef.current.traceId,
...attributes,
});
} catch {
// No-op. Logging must never throw into the UI.
}
},
[],
);
}

View File

@@ -20,7 +20,10 @@ import {
} from 'types/api/trace/getTraceV3';
import { COLOR_BY_FIELDS } from './constants';
import { TraceDetailEventKeys, TraceDetailEvents } from './events';
import { useTraceDetailLogEvent } from './hooks/useTraceDetailLogEvent';
import TraceStoreSync from './stores/TraceStoreSync';
import { useTraceStore } from './stores/traceStore';
import { AGGREGATIONS } from './utils/aggregations';
import { SpanDetailVariant } from './SpanDetailsPanel/constants';
import SpanDetailsPanel from './SpanDetailsPanel/SpanDetailsPanel';
@@ -56,6 +59,14 @@ function TraceDetailsV3(): JSX.Element {
const selectedSpanId = urlQuery.get('spanId') || undefined;
const { safeNavigate } = useSafeNavigate();
const logTraceEvent = useTraceDetailLogEvent('v3', traceId || '');
// Tracks which traceId the load event already fired for, so navigating
// between traces (the route component stays mounted) re-fires it once each.
const dataLoadedFiredForRef = useRef('');
const colorByField = useTraceStore((s) => s.colorByField);
const previewFieldsCount = useTraceStore((s) => s.previewFields.length);
const userPrefsReady = useTraceStore((s) => s.userPreferences !== null);
const handleSpanDetailsClose = useCallback((): void => {
urlQuery.delete('spanId');
safeNavigate({ search: urlQuery.toString() });
@@ -154,6 +165,46 @@ function TraceDetailsV3(): JSX.Element {
allSpansRef.current = allSpans;
}, [allSpans]);
useEffect(() => {
if (
!traceId ||
dataLoadedFiredForRef.current === traceId ||
!userPrefsReady
) {
return;
}
const payload = traceData?.payload;
if (!payload?.spans?.length) {
return;
}
dataLoadedFiredForRef.current = traceId;
const numServices = new Set(payload.spans.map((s) => s['service.name'])).size;
logTraceEvent(TraceDetailEvents.DataLoaded, {
[TraceDetailEventKeys.TotalSpansCount]: totalSpansCount,
[TraceDetailEventKeys.NumServices]: numServices,
[TraceDetailEventKeys.TraceDurationMs]:
payload.endTimestampMillis - payload.startTimestampMillis,
[TraceDetailEventKeys.HadErrors]: (payload.totalErrorSpansCount || 0) > 0,
[TraceDetailEventKeys.FlamegraphSampled]:
totalSpansCount > FLAMEGRAPH_SPAN_LIMIT,
[TraceDetailEventKeys.SpanPanelVariant]:
getLocalStorageKey(LOCALSTORAGE.TRACE_DETAILS_SPAN_DETAILS_POSITION) ||
SpanDetailVariant.DOCKED_RIGHT,
[TraceDetailEventKeys.ColorByField]: colorByField.name,
[TraceDetailEventKeys.PreviewFieldsCount]: previewFieldsCount,
[TraceDetailEventKeys.EntryPreferOldView]:
getLocalStorageKey(LOCALSTORAGE.TRACE_DETAILS_PREFER_OLD_VIEW) === 'true',
});
}, [
traceId,
userPrefsReady,
traceData,
totalSpansCount,
colorByField,
previewFieldsCount,
logTraceEvent,
]);
// Frontend mode: expand all parents by default when full data arrives
useEffect(() => {
if (isFullDataLoaded && allSpans.length > 0) {
@@ -233,6 +284,12 @@ function TraceDetailsV3(): JSX.Element {
const [activeKeys, setActiveKeys] = useState<string[]>(['flame', 'waterfall']);
const handleCollapseChange = (key: string): void => {
logTraceEvent(
key === 'flame'
? TraceDetailEvents.FlameGraphToggled
: TraceDetailEvents.WaterfallToggled,
{ [TraceDetailEventKeys.Expanded]: !activeKeys.includes(key) },
);
setActiveKeys((prev) =>
prev.includes(key) ? prev.filter((k) => k !== key) : [...prev, key],
);

View File

@@ -4,6 +4,7 @@ import { ChevronDown, Copy } from '@signozhq/icons';
import { Button } from '@signozhq/ui/button';
import { DropdownMenuSimple as Dropdown } from '@signozhq/ui/dropdown-menu';
import { toast } from '@signozhq/ui/sonner';
import logEvent from 'api/common/logEvent';
import { JsonView } from 'periscope/components/JsonView';
import { PrettyView } from 'periscope/components/PrettyView';
import { PrettyViewProps } from 'periscope/components/PrettyView';
@@ -12,6 +13,8 @@ import './DataViewer.styles.scss';
type ViewMode = 'pretty' | 'json';
const VIEW_MODE_CHANGED_EVENT = 'Data Viewer: View mode changed';
const VIEW_MODE_OPTIONS: { label: string; value: ViewMode }[] = [
{ label: 'Pretty', value: 'pretty' },
{ label: 'JSON', value: 'json' },
@@ -34,6 +37,20 @@ function DataViewer({
const jsonString = useMemo(() => JSON.stringify(data, null, 2), [data]);
const handleViewModeChange = (value: string): void => {
const next = value as ViewMode;
setViewMode(next);
try {
logEvent(VIEW_MODE_CHANGED_EVENT, {
viewMode: next,
path: window.location.pathname,
drawerKey,
});
} catch {
// No op
}
};
const handleCopy = (): void => {
const text = JSON.stringify(data, null, 2);
setCopy(text);
@@ -56,7 +73,7 @@ function DataViewer({
{
type: 'radio-group',
value: viewMode,
onChange: (value): void => setViewMode(value as ViewMode),
onChange: handleViewModeChange,
children: VIEW_MODE_OPTIONS.map((opt) => ({
type: 'radio',
key: opt.value,

View File

@@ -67,5 +67,24 @@ func (provider *provider) addTraceDetailRoutes(router *mux.Router) error {
return err
}
if err := router.Handle("/api/v3/traces/{traceID}/flamegraph", handler.New(
provider.authzMiddleware.ViewAccess(provider.traceDetailHandler.GetFlamegraph),
handler.OpenAPIDef{
ID: "GetFlamegraph",
Tags: []string{"tracedetail"},
Summary: "Get flamegraph view for a trace",
Description: "Returns the flamegraph view of spans for a given trace ID.",
Request: new(spantypes.PostableFlamegraph),
RequestContentType: "application/json",
Response: new(spantypes.GettableFlamegraphTrace),
ResponseContentType: "application/json",
SuccessStatusCode: http.StatusOK,
ErrorStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound},
SecuritySchemes: newSecuritySchemes(types.RoleViewer),
},
)).Methods(http.MethodPost).GetError(); err != nil {
return err
}
return nil
}

View File

@@ -6,7 +6,16 @@ import (
)
type Config struct {
Waterfall WaterfallConfig `mapstructure:"waterfall"`
Waterfall WaterfallConfig `mapstructure:"waterfall"`
Flamegraph FlamegraphConfig `mapstructure:"flamegraph"`
}
type FlamegraphConfig struct {
MaxSelectedLevels int `mapstructure:"max_selected_levels"`
MaxSpansPerLevel int `mapstructure:"max_spans_per_level"`
SamplingTopLatencySpansCount int `mapstructure:"sampling_top_latency_count"`
SamplingBucketCount int `mapstructure:"sampling_bucket_count"`
SelectAllSpansLimit uint `mapstructure:"select_all_spans_limit"`
}
type WaterfallConfig struct {
@@ -29,6 +38,13 @@ func newConfig() factory.Config {
MaxDepthToAutoExpand: 5,
MaxLimitToSelectAllSpans: 10_000,
},
Flamegraph: FlamegraphConfig{
MaxSelectedLevels: 50,
MaxSpansPerLevel: 100,
SamplingTopLatencySpansCount: 5,
SamplingBucketCount: 50,
SelectAllSpansLimit: 100_000,
},
}
}
@@ -45,5 +61,25 @@ func (c Config) Validate() error {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"tracedetail.waterfall.max_limit_to_select_all_spans must be positive")
}
if c.Flamegraph.MaxSelectedLevels <= 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"tracedetail.flamegraph.level_limit must be positive, got %d", c.Flamegraph.MaxSelectedLevels)
}
if c.Flamegraph.MaxSpansPerLevel <= 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"tracedetail.flamegraph.spans_per_level must be positive, got %d", c.Flamegraph.MaxSpansPerLevel)
}
if c.Flamegraph.SamplingTopLatencySpansCount < 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"tracedetail.flamegraph.top_latency_count cannot be negative, got %d", c.Flamegraph.SamplingTopLatencySpansCount)
}
if c.Flamegraph.SamplingBucketCount <= 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"tracedetail.flamegraph.bucket_count must be positive, got %d", c.Flamegraph.SamplingBucketCount)
}
if c.Flamegraph.SelectAllSpansLimit == 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"tracedetail.flamegraph.max_limit_to_select_all_spans must be positive")
}
return nil
}

View File

@@ -80,3 +80,19 @@ func (h *handler) GetTraceAggregations(rw http.ResponseWriter, r *http.Request)
render.Success(rw, http.StatusOK, result)
}
func (h *handler) GetFlamegraph(rw http.ResponseWriter, r *http.Request) {
req := new(spantypes.PostableFlamegraph)
if err := binding.JSON.BindBody(r.Body, req); err != nil {
render.Error(rw, err)
return
}
result, err := h.module.GetFlamegraph(r.Context(), mux.Vars(r)["traceID"], req)
if err != nil {
render.Error(rw, err)
return
}
render.Success(rw, http.StatusOK, result)
}

View File

@@ -110,7 +110,6 @@ func (m *module) GetTraceAggregations(ctx context.Context, traceID string, req *
if err != nil {
return nil, err
}
traceDurationNs := uint64(summary.End.UnixNano()) - uint64(summary.Start.UnixNano())
results := make([]spantypes.SpanAggregationResult, 0, len(req.Aggregations))
@@ -148,6 +147,17 @@ func (m *module) GetTraceAggregations(ctx context.Context, traceID string, req *
return &spantypes.GettableTraceAggregations{Aggregations: results}, nil
}
func (m *module) GetFlamegraph(ctx context.Context, traceID string, req *spantypes.PostableFlamegraph) (*spantypes.GettableFlamegraphTrace, error) {
summary, err := m.store.GetTraceSummary(ctx, traceID)
if err != nil {
return nil, err
}
if summary.NumSpans <= uint64(m.config.Flamegraph.SelectAllSpansLimit) {
return m.getFullFlamegraph(ctx, traceID, summary)
}
return m.getWindowedFlamegraph(ctx, traceID, req.SelectedSpanID, summary)
}
// getWindowedWaterfall builds the waterfall tree with minimal data and then returns only a window of full spans.
func (m *module) getWindowedWaterfall(ctx context.Context, traceID, selectedSpanID string, uncollapsedSpans []string, start, end time.Time) (*spantypes.GettableWaterfallTrace, error) {
// Step 1: minimal fetch → build full tree → select visible window
@@ -188,3 +198,50 @@ func (m *module) getWindowedWaterfall(ctx context.Context, traceID, selectedSpan
waterfallTrace, selectedSpans, uncollapsedSpans, false, nil,
), nil
}
func (m *module) getFullFlamegraph(ctx context.Context, traceID string, summary *spantypes.TraceSummary) (*spantypes.GettableFlamegraphTrace, error) {
fullSpans, err := m.store.GetTraceSpans(ctx, traceID, summary)
if err != nil {
return nil, err
}
if len(fullSpans) == 0 {
return nil, spantypes.ErrTraceNotFound
}
flamegraphTrace := spantypes.NewFlamegraphTraceFromStorable(fullSpans)
return spantypes.NewGettableFlamegraphTrace(
flamegraphTrace.GetAllLevels(),
summary.Start.UnixMilli(), summary.End.UnixMilli(), false,
), nil
}
// getWindowedFlamegraph returns a window of a max levels and max sampled spans per level around the selected span.
func (m *module) getWindowedFlamegraph(ctx context.Context, traceID, selectedSpanID string, summary *spantypes.TraceSummary) (*spantypes.GettableFlamegraphTrace, error) {
minimalSpans, err := m.store.GetMinimalSpans(ctx, traceID, summary.Start, summary.End)
if err != nil {
return nil, err
}
if len(minimalSpans) == 0 {
return nil, spantypes.ErrTraceNotFound
}
flamegraphTrace := spantypes.NewFlamegraphTraceFromMinimal(minimalSpans)
minimalSpans = nil //nolint:ineffassign,wastedassign // release backing array before further db calls
cfg := m.config.Flamegraph
selectedSpans := flamegraphTrace.GetSelectedLevels(selectedSpanID,
cfg.MaxSelectedLevels, cfg.MaxSpansPerLevel, cfg.SamplingTopLatencySpansCount, cfg.SamplingBucketCount)
if len(selectedSpans) == 0 {
return nil, spantypes.ErrTraceNotFound
}
fullSpans, err := m.store.GetTraceSpansByIDs(ctx, traceID, summary.Start, summary.End,
spantypes.FlamegraphWindowSpanIDs(selectedSpans))
if err != nil {
return nil, err
}
return spantypes.NewGettableFlamegraphTrace(
flamegraphTrace.EnrichSelectedSpans(selectedSpans, fullSpans),
summary.Start.UnixMilli(), summary.End.UnixMilli(), true,
), nil
}

View File

@@ -74,7 +74,7 @@ func (s *traceStore) GetTraceSpans(ctx context.Context, traceID string, summary
events, status_message, status_code_string, kind_string, parent_span_id,
flags, is_remote, trace_state, status_code,
db_name, db_operation, http_method, http_url, http_host,
external_http_method, external_http_url, response_status_code
external_http_method, external_http_url, response_status_code, links as references
FROM %s.%s
WHERE trace_id=? AND ts_bucket_start>=? AND ts_bucket_start<=?
ORDER BY timestamp ASC, name ASC`,
@@ -130,7 +130,7 @@ func (s *traceStore) GetTraceSpansByIDs(ctx context.Context, traceID string, sta
"events", "status_message", "status_code_string", "kind_string", "parent_span_id",
"flags", "is_remote", "trace_state", "status_code",
"db_name", "db_operation", "http_method", "http_url", "http_host",
"external_http_method", "external_http_url", "response_status_code",
"external_http_method", "external_http_url", "response_status_code", "links as references",
)
sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
ids := make([]any, len(spanIDs))

View File

@@ -12,6 +12,7 @@ type Handler interface {
GetWaterfall(http.ResponseWriter, *http.Request)
GetWaterfallV4(http.ResponseWriter, *http.Request)
GetTraceAggregations(http.ResponseWriter, *http.Request)
GetFlamegraph(http.ResponseWriter, *http.Request)
}
// Module defines the business logic for trace detail operations.
@@ -19,4 +20,5 @@ type Module interface {
GetWaterfall(ctx context.Context, traceID string, req *spantypes.PostableWaterfall) (*spantypes.GettableWaterfallTrace, error)
GetWaterfallV4(ctx context.Context, traceID string, selectedSpanID string, uncollapsedSpans []string, selectAllLimit uint) (*spantypes.GettableWaterfallTrace, error)
GetTraceAggregations(ctx context.Context, traceID string, req *spantypes.PostableTraceAggregations) (*spantypes.GettableTraceAggregations, error)
GetFlamegraph(ctx context.Context, traceID string, req *spantypes.PostableFlamegraph) (*spantypes.GettableFlamegraphTrace, error)
}

View File

@@ -0,0 +1,81 @@
package spantypes
import (
"maps"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
type FlamegraphSpan struct {
SpanID string `json:"spanId"`
ParentSpanID string `json:"parentSpanId"`
Timestamp uint64 `json:"timestamp"`
DurationNano uint64 `json:"durationNano"`
HasError bool `json:"hasError"`
ServiceName string `json:"serviceName"`
Name string `json:"name"`
Level int64 `json:"level"`
Events []Event `json:"event"`
Attributes map[string]any `json:"attributes,omitempty"`
Resource map[string]string `json:"resource,omitempty"`
Children []*FlamegraphSpan `json:"-"` // internal tree use only
}
// FlamegraphLevel groups span IDs at a single level within the selected window.
type FlamegraphLevel struct {
Level int64
SpanIDs []string
}
type PostableFlamegraph struct {
SelectedSpanID string `json:"selectedSpanId"`
SelectFields []telemetrytypes.TelemetryFieldKey `json:"selectFields,omitempty"`
}
// GettableFlamegraphTrace is the response for the v3 flamegraph API.
type GettableFlamegraphTrace struct {
Spans [][]*FlamegraphSpan `json:"spans"`
StartTimestampMillis int64 `json:"startTimestampMillis"`
EndTimestampMillis int64 `json:"endTimestampMillis"`
HasMore bool `json:"hasMore"`
}
func NewGettableFlamegraphTrace(spans [][]*FlamegraphSpan, startMs, endMs int64, hasMore bool) *GettableFlamegraphTrace {
return &GettableFlamegraphTrace{
Spans: spans,
StartTimestampMillis: startMs,
EndTimestampMillis: endMs,
HasMore: hasMore,
}
}
func NewFlamegraphSpanFromStorable(s *StorableSpan, level int64) *FlamegraphSpan {
resources := make(map[string]string, len(s.ResourcesString))
maps.Copy(resources, s.ResourcesString)
return &FlamegraphSpan{
SpanID: s.SpanID,
ParentSpanID: s.ParentSpanID,
Timestamp: uint64(s.StartTime.UnixNano()),
DurationNano: s.DurationNano,
HasError: s.HasError,
ServiceName: s.ServiceName,
Name: s.Name,
Level: level,
Events: s.UnmarshalledEvents(),
Attributes: s.Attributes(),
Resource: resources,
}
}
// FlamegraphWindowSpanIDs collects all span IDs from a level window into a flat slice.
func FlamegraphWindowSpanIDs(window []FlamegraphLevel) []string {
total := 0
for _, lvl := range window {
total += len(lvl.SpanIDs)
}
ids := make([]string, 0, total)
for _, lvl := range window {
ids = append(ids, lvl.SpanIDs...)
}
return ids
}

View File

@@ -0,0 +1,118 @@
package spantypes
import (
"sort"
)
// FlamegraphTrace holds the level wise tree built from minimal spans.
type FlamegraphTrace struct {
roots []*FlamegraphSpan
nodeByID map[string]*FlamegraphSpan
startTime uint64
endTime uint64
}
func NewFlamegraphTraceFromMinimal(spans []MinimalSpan) *FlamegraphTrace {
t := &FlamegraphTrace{
nodeByID: make(map[string]*FlamegraphSpan, len(spans)),
}
for i := range spans {
node := spans[i].ToFlamegraphSpan()
t.updateTimeRange(node.Timestamp, node.DurationNano)
t.nodeByID[node.SpanID] = node
}
t.buildSpanTree()
return t
}
func NewFlamegraphTraceFromStorable(spans []StorableSpan) *FlamegraphTrace {
t := &FlamegraphTrace{
nodeByID: make(map[string]*FlamegraphSpan, len(spans)),
}
for i := range spans {
node := NewFlamegraphSpanFromStorable(&spans[i], 0) // level is set later by BFS
t.updateTimeRange(node.Timestamp, node.DurationNano)
t.nodeByID[node.SpanID] = node
}
t.buildSpanTree()
return t
}
func (t *FlamegraphTrace) GetAllLevels() [][]*FlamegraphSpan {
return nil
}
// GetSelectedLevels returns the window of levels around selectedSpanID with sampling applied to dense levels.
func (t *FlamegraphTrace) GetSelectedLevels(
selectedSpanID string,
levelLimit, spansPerLevel, topLatencyCount, bucketCount int,
) []FlamegraphLevel {
return nil
}
func (t *FlamegraphTrace) EnrichSelectedSpans(selectedSpans []FlamegraphLevel, fullSpans []StorableSpan) [][]*FlamegraphSpan {
fullByID := make(map[string]*StorableSpan, len(fullSpans))
for i := range fullSpans {
fullByID[fullSpans[i].SpanID] = &fullSpans[i]
}
result := make([][]*FlamegraphSpan, len(selectedSpans))
for i, lvl := range selectedSpans {
result[i] = make([]*FlamegraphSpan, 0, len(lvl.SpanIDs))
for _, spanID := range lvl.SpanIDs {
if full, ok := fullByID[spanID]; ok {
result[i] = append(result[i], NewFlamegraphSpanFromStorable(full, lvl.Level))
} else if lean, ok := t.nodeByID[spanID]; ok {
result[i] = append(result[i], lean)
}
}
}
return result
}
func (t *FlamegraphTrace) updateTimeRange(timestamp, durationNano uint64) {
if t.startTime == 0 || timestamp < t.startTime {
t.startTime = timestamp
}
if end := timestamp + durationNano; end > t.endTime {
t.endTime = end
}
}
func (t *FlamegraphTrace) buildSpanTree() {
for _, node := range t.nodeByID {
if node.ParentSpanID != "" {
if parent, ok := t.nodeByID[node.ParentSpanID]; ok {
parent.Children = append(parent.Children, node)
} else {
missing := &FlamegraphSpan{
SpanID: node.ParentSpanID,
Name: "Missing Span",
Timestamp: node.Timestamp,
DurationNano: node.DurationNano,
Children: []*FlamegraphSpan{node},
}
t.nodeByID[missing.SpanID] = missing
t.roots = append(t.roots, missing)
}
} else if flamegraphSpanIndex(t.roots, node.SpanID) == -1 {
t.roots = append(t.roots, node)
}
}
sort.Slice(t.roots, func(i, j int) bool {
if t.roots[i].Timestamp == t.roots[j].Timestamp {
return t.roots[i].SpanID < t.roots[j].SpanID
}
return t.roots[i].Timestamp < t.roots[j].Timestamp
})
}
func flamegraphSpanIndex(spans []*FlamegraphSpan, spanID string) int {
for i, s := range spans {
if s != nil && s.SpanID == spanID {
return i
}
}
return -1
}

View File

@@ -54,6 +54,12 @@ type Event struct {
IsError bool `json:"isError,omitempty"`
}
type OtelSpanRef struct {
TraceId string `json:"traceId,omitempty"`
SpanId string `json:"spanId,omitempty"`
RefType string `json:"refType,omitempty"`
}
// WaterfallSpan represents the span in waterfall response,
// this uses snake_case keys for response as a special case since these
// keys can be directly used to query spans and client need to know the actual fields.
@@ -74,6 +80,7 @@ type WaterfallSpan struct {
TimeUnix uint64 `json:"time_unix"`
TraceID string `json:"trace_id"`
TraceState string `json:"trace_state"`
References []OtelSpanRef `json:"references" required:"true" nullable:"false"`
// Calculated fields https://signoz.io/docs/traces-management/guides/derived-fields-spans
DBName string `json:"db_name,omitempty"`
@@ -128,6 +135,7 @@ type StorableSpan struct {
ExternalHTTPMethod string `ch:"external_http_method"`
ExternalHTTPURL string `ch:"external_http_url"`
ResponseStatusCode string `ch:"response_status_code"`
References string `ch:"references"`
}
// MinimalSpan with only the fields needed to build the parent-child tree.
@@ -156,6 +164,18 @@ func (item *MinimalSpan) ToWaterfallSpan(traceID string) *WaterfallSpan {
}
}
func (item *MinimalSpan) ToFlamegraphSpan() *FlamegraphSpan {
return &FlamegraphSpan{
SpanID: item.SpanID,
ParentSpanID: item.ParentSpanID,
Timestamp: uint64(item.StartTime.UnixNano()),
DurationNano: item.DurationNano,
HasError: item.HasError,
ServiceName: item.ServiceName,
Children: make([]*FlamegraphSpan, 0),
}
}
// NewMissingWaterfallSpan creates a synthetic placeholder span for a parent that has no recorded data.
func NewMissingWaterfallSpan(spanID, traceID string, timeUnixNano, durationNano uint64) *WaterfallSpan {
return &WaterfallSpan{
@@ -285,6 +305,14 @@ func (item *StorableSpan) UnmarshalledEvents() []Event {
return events
}
func (item *StorableSpan) UnmarshalledRefs() []OtelSpanRef {
refs := []OtelSpanRef{}
if err := json.Unmarshal([]byte(item.References), &refs); err != nil {
return nil // skip malformed values
}
return refs
}
func (item *StorableSpan) ToWaterfallSpan(traceID string) *WaterfallSpan {
resources := make(map[string]string)
maps.Copy(resources, item.ResourcesString)
@@ -318,6 +346,7 @@ func (item *StorableSpan) ToWaterfallSpan(traceID string) *WaterfallSpan {
Children: make([]*WaterfallSpan, 0),
TimeUnix: uint64(item.StartTime.UnixNano()),
ServiceName: item.ServiceName,
References: item.UnmarshalledRefs(),
}
}

View File

@@ -54,16 +54,16 @@ func newConfig() factory.Config {
Directory: "/etc/signoz/web",
Settings: SettingsConfig{
Posthog: PosthogConfig{
Enabled: true,
Enabled: false,
},
Appcues: AppcuesConfig{
Enabled: true,
Enabled: false,
},
Sentry: SentryConfig{
Enabled: true,
Enabled: false,
},
Pylon: PylonConfig{
Enabled: true,
Enabled: false,
},
},
}