Compare commits

..

13 Commits

Author SHA1 Message Date
aks07
4ec3754b65 feat: increase limit 2026-02-26 17:45:21 +05:30
aks07
c36f780964 feat: poc flamegraph 3 2026-02-26 15:57:41 +05:30
aks07
e0e0870e23 feat: poc flamegraph 2 2026-02-26 15:41:15 +05:30
aks07
ea406dac52 feat: poc flamegraph 1 2026-02-26 15:40:36 +05:30
aks07
c6152b33e9 feat: poc flamegraph 2026-02-26 15:37:28 +05:30
Nikhil Soni
ca87244961 feat: increase fg limits and add timestamp boundaries 2026-02-25 18:27:12 +05:30
Nikhil Soni
e41a85febd feat: return all spans for flamegraph under a limit 2026-02-25 11:23:58 +05:30
Nikhil Soni
9d4154a117 fix: add missing filtering for ip address for scalar data (#10264)
* fix: add missing filtering for ip address for scalar data

In domain listing api for external api monitoring,
we have option to filter out the IP address but
it only handles timeseries and raw type data while
domain list handler returns scalar data.

* fix: switch to new derived attributes for ip filtering

---------

Co-authored-by: Nityananda Gohain <nityanandagohain@gmail.com>
2026-02-25 11:23:58 +05:30
Nikhil Soni
4b9dd63e8c Revert "feat: show spinner on expanding span and fix autoscroll"
This reverts commit 6bd194dfb4.
2026-02-24 19:53:00 +05:30
Nikhil Soni
30e7736a74 feat: add support for returning all the spans
If total spans are less than a max value.
This allows UI to not have to call API
for smaller spans on ever collapse/uncollapse
2026-02-24 19:48:34 +05:30
Nikhil Soni
6bd194dfb4 feat: show spinner on expanding span and fix autoscroll
Two improvemnts are included in this commit:
- Show a loading spiner in UI when a span is expanded
  to signal api call.
- Don't auto scroll the waterfall to bring the selected
  span to centre in case it's expanded or scrolled up
  because it looks very flickering behaviour to user
2026-02-24 15:24:57 +05:30
Nikhil Soni
f27e3cabb2 feat: add support for configurable pre expanded levels
When a span is expanded, now it will return nested children
up to a max depth level
2026-02-24 15:24:57 +05:30
Piyush Singariya
bc4273f2f8 chore: test clickhouse version 25.12.5 (#10402) 2026-02-24 14:55:51 +05:30
29 changed files with 6493 additions and 1915 deletions

View File

@@ -54,7 +54,7 @@ jobs:
- sqlite
clickhouse-version:
- 25.5.6
- 25.10.5
- 25.12.5
schema-migrator-version:
- v0.142.0
postgres-version:

View File

@@ -14,10 +14,16 @@ interface ITimelineV2Props {
startTimestamp: number;
endTimestamp: number;
timelineHeight: number;
offsetTimestamp: number;
}
function TimelineV2(props: ITimelineV2Props): JSX.Element {
const { startTimestamp, endTimestamp, timelineHeight } = props;
const {
startTimestamp,
endTimestamp,
timelineHeight,
offsetTimestamp,
} = props;
const [intervals, setIntervals] = useState<Interval[]>([]);
const [ref, { width }] = useMeasure<HTMLDivElement>();
const isDarkMode = useIsDarkMode();
@@ -30,8 +36,10 @@ function TimelineV2(props: ITimelineV2Props): JSX.Element {
const minIntervals = getMinimumIntervalsBasedOnWidth(width);
const intervalisedSpread = (spread / minIntervals) * 1.0;
setIntervals(getIntervals(intervalisedSpread, spread));
}, [startTimestamp, endTimestamp, width]);
const intervals = getIntervals(intervalisedSpread, spread, offsetTimestamp);
setIntervals(intervals);
}, [startTimestamp, endTimestamp, width, offsetTimestamp]);
if (endTimestamp < startTimestamp) {
console.error(

View File

@@ -64,6 +64,71 @@ export const resolveTimeFromInterval = (
export function getIntervals(
intervalSpread: number,
baseSpread: number,
offsetTimestamp: number, // ms offset from trace start (e.g. viewStart - traceStart)
): Interval[] {
const integerPartString = intervalSpread.toString().split('.')[0];
const integerPartLength = integerPartString.length;
const intervalSpreadNormalized =
intervalSpread < 1.0
? intervalSpread
: Math.floor(Number(integerPartString) / 10 ** (integerPartLength - 1)) *
10 ** (integerPartLength - 1);
let intervalUnit = INTERVAL_UNITS[0];
for (let idx = INTERVAL_UNITS.length - 1; idx >= 0; idx -= 1) {
const standardInterval = INTERVAL_UNITS[idx];
if (intervalSpread * standardInterval.multiplier >= 1) {
intervalUnit = INTERVAL_UNITS[idx];
break;
}
}
const intervals: Interval[] = [
{
// ✅ start label should reflect window start offset (relative to trace start)
label: `${toFixed(
resolveTimeFromInterval(offsetTimestamp, intervalUnit),
2,
)}${intervalUnit.name}`,
percentage: 0,
},
];
let tempBaseSpread = baseSpread;
let elapsedIntervals = 0;
while (tempBaseSpread && intervals.length < 20) {
let intervalTime: number;
if (tempBaseSpread <= 1.5 * intervalSpreadNormalized) {
intervalTime = elapsedIntervals + tempBaseSpread;
tempBaseSpread = 0;
} else {
intervalTime = elapsedIntervals + intervalSpreadNormalized;
tempBaseSpread -= intervalSpreadNormalized;
}
elapsedIntervals = intervalTime;
// ✅ label time = window offset + elapsed time inside window
const labelTime = offsetTimestamp + intervalTime;
intervals.push({
label: `${toFixed(resolveTimeFromInterval(labelTime, intervalUnit), 2)}${
intervalUnit.name
}`,
percentage: (intervalTime / baseSpread) * 100,
});
}
return intervals;
}
export function getIntervalsOld(
intervalSpread: number,
baseSpread: number,
offsetTimestamp: number,
): Interval[] {
const integerPartString = intervalSpread.toString().split('.')[0];
const integerPartLength = integerPartString.length;
@@ -106,9 +171,10 @@ export function getIntervals(
}
elapsedIntervals = intervalTime;
const interval: Interval = {
label: `${toFixed(resolveTimeFromInterval(intervalTime, intervalUnit), 2)}${
intervalUnit.name
}`,
label: `${toFixed(
resolveTimeFromInterval(intervalTime + offsetTimestamp, intervalUnit),
2,
)}${intervalUnit.name}`,
percentage: (intervalTime / baseSpread) * 100,
};
intervals.push(interval);

View File

@@ -0,0 +1,88 @@
# Flamegraph Canvas POC Notes
## Overview
This document tracks the proof-of-concept (POC) implementation of a canvas-based flamegraph rendering system, replacing the previous DOM-based approach.
## Implementation Status
### ✅ Completed Features
1. **Canvas-based Rendering**
- Replaced DOM elements (`react-virtuoso` and `div.span-item`) with a single HTML5 Canvas
- Implemented `drawFlamegraph` function to render all spans as rectangles on canvas
- Added device pixel ratio (DPR) support for crisp rendering
2. **Time-window-based Zoom**
- Replaced pixel-based zoom/pan with time-window-based approach (`viewStartTs`, `viewEndTs`)
- Prevents pixelation by redrawing from data with new time bounds
- Zoom anchors to cursor position
- Horizontal zoom works correctly (min: 1/100th of trace, max: full trace)
3. **Drag to Pan**
- Implemented drag-to-pan functionality for navigating the canvas
- Differentiates between click (span selection) and drag (panning) based on distance moved
- Prevents unwanted window zoom
4. **Minimap with 2D Navigation**
- Canvas-based minimap showing density histogram (time × levels)
- 2D brush overlay for both horizontal (time) and vertical (levels) navigation
- Draggable brush to pan both dimensions
- Bidirectional synchronization between main canvas and minimap
5. **Timeline Synchronization**
- `TimelineV2` component synchronized with visible time window
- Updates correctly during zoom and pan operations
6. **Hit Testing**
- Implemented span rectangle tracking for click detection
- Tooltip on hover
- Span selection via click
### ❌ Known Issues / Not Working
1. **Vertical Zoom - NOT WORKING**
- **Status**: Attempted implementation but not functioning correctly
- **Issue**: When horizontal zoom reaches maximum (full trace width), vertical zoom cannot continue to zoom out further
- **Attempted Solution**: Added `rowHeightScale` state to control vertical row spacing, but the implementation does not work as expected
- **Impact**: Users cannot fully zoom out vertically to see all levels when horizontal zoom is at maximum
- **Next Steps**: Needs further investigation and alternative approach
2. **Timeline Scale Alignment - NOT WORKING PROPERLY**
- **Status**: Issue identified but not fully resolved
- **Issue**: The timeline scale does not align properly when dragging/panning the canvas. The timeline aligns correctly during zoom operations, but not during drag/pan operations.
- **Impact**: Timeline may show incorrect time values while dragging the canvas
- **Attempted Solution**: Used refs (`viewStartTsRef`, `viewEndTsRef`) to track current time window and incremental delta calculation, but issue persists
- **Next Steps**: Needs further investigation to ensure timeline stays synchronized during all interaction types
### 🔄 Pending / Future Work
1. **Performance Optimization**
- Consider adding an interaction layer (separate canvas on top) for better performance
- Optimize rendering for large traces
2. **Code Quality**
- Reduce cognitive complexity of `drawFlamegraph` function (currently 26, target: 15)
- Reduce cognitive complexity of `drawMinimap` function (currently 30, target: 15)
3. **Additional Features**
- Keyboard shortcuts for navigation
- Better zoom controls
- Export functionality
## Technical Details
### Key Files Modified
- `frontend/src/container/PaginatedTraceFlamegraph/TraceFlamegraphStates/Success/Success.tsx` - Main rendering component
- `frontend/src/container/PaginatedTraceFlamegraph/TraceFlamegraphStates/Success/Success.styles.scss` - Styles for canvas and minimap
- `frontend/src/components/TimelineV2/TimelineV2.tsx` - Timeline synchronization
### Key Concepts
- **Time-window-based zoom**: Instead of scaling canvas bitmap, redraw from data with new time bounds
- **Device Pixel Ratio**: Render at DPR resolution for crisp display on high-DPI screens
- **2D Minimap**: Shows density heatmap across both time (horizontal) and levels (vertical) dimensions
- **Brush Navigation**: Draggable rectangle overlay for panning both dimensions
## Notes
- This is a POC implementation - code quality and optimization can be improved after validation
- Some linting warnings (cognitive complexity) are acceptable for POC phase
- All changes should be validated before production use

View File

@@ -1,3 +1,5 @@
// @ts-nocheck
/* eslint-disable */
import { useEffect, useMemo, useState } from 'react';
import { useParams } from 'react-router-dom';
import { Progress, Skeleton, Tooltip, Typography } from 'antd';
@@ -13,8 +15,13 @@ import { Span } from 'types/api/trace/getTraceV2';
import { TraceFlamegraphStates } from './constants';
import Error from './TraceFlamegraphStates/Error/Error';
import NoData from './TraceFlamegraphStates/NoData/NoData';
import Success from './TraceFlamegraphStates/Success/Success';
// import Success from './TraceFlamegraphStates/Success/SuccessV2';
// import Success from './TraceFlamegraphStates/Success/SuccessV3_without_minimap_best';
import Success from './TraceFlamegraphStates/Success/Success_zoom';
// import Success from './TraceFlamegraphStates/Success/Success_zoom_api';
// import Success from './TraceFlamegraphStates/Success/SuccessCursor';
// import Success from './TraceFlamegraphStates/Success/Success';
import './PaginatedTraceFlamegraph.styles.scss';
interface ITraceFlamegraphProps {
@@ -38,7 +45,6 @@ function TraceFlamegraph(props: ITraceFlamegraphProps): JSX.Element {
const [firstSpanAtFetchLevel, setFirstSpanAtFetchLevel] = useState<string>(
urlQuery.get('spanId') || '',
);
useEffect(() => {
setFirstSpanAtFetchLevel(urlQuery.get('spanId') || '');
}, [urlQuery]);
@@ -46,6 +52,9 @@ function TraceFlamegraph(props: ITraceFlamegraphProps): JSX.Element {
const { data, isFetching, error } = useGetTraceFlamegraph({
traceId,
selectedSpanId: firstSpanAtFetchLevel,
limit: 100001,
// boundaryStartTsMilli: 0,
// boundarEndTsMilli: 10000,
});
// get the current state of trace flamegraph based on the API lifecycle

View File

@@ -3,6 +3,11 @@
overflow-x: hidden;
overflow-y: auto;
&.trace-flamegraph-canvas {
overflow: hidden;
position: relative;
}
.trace-flamegraph-virtuoso {
overflow-x: hidden;

View File

@@ -1,3 +1,6 @@
// @ts-nocheck
/* eslint-disable */
/* eslint-disable jsx-a11y/no-static-element-interactions */
/* eslint-disable jsx-a11y/click-events-have-key-events */
import {
@@ -23,6 +26,12 @@ import { toFixed } from 'utils/toFixed';
import './Success.styles.scss';
// Constants for rendering
const ROW_HEIGHT = 24; // 18px height + 6px padding
const SPAN_BAR_HEIGHT = 12;
const EVENT_DOT_SIZE = 6;
const SPAN_BAR_Y_OFFSET = 3; //
interface ITraceMetadata {
startTime: number;
endTime: number;
@@ -35,6 +44,14 @@ interface ISuccessProps {
traceMetadata: ITraceMetadata;
selectedSpan: Span | undefined;
}
interface SpanRect {
span: FlamegraphSpan;
x: number;
y: number;
width: number;
height: number;
level: number;
}
function Success(props: ISuccessProps): JSX.Element {
const {
@@ -48,6 +65,28 @@ function Success(props: ISuccessProps): JSX.Element {
const history = useHistory();
const isDarkMode = useIsDarkMode();
const virtuosoRef = useRef<VirtuosoHandle>(null);
const baseCanvasRef = useRef<HTMLCanvasElement>(null);
const interactionCanvasRef = useRef<HTMLCanvasElement>(null);
const minimapRef = useRef<HTMLCanvasElement>(null);
const containerRef = useRef<HTMLDivElement>(null);
console.log('spans.length', spans.length);
// Calculate total canvas height. this is coming less
const totalHeight = spans.length * ROW_HEIGHT;
// Build a flat array of span rectangles for hit testing.
// consider per level buckets to improve hit testing
const spanRects = useRef<SpanRect[]>([]);
// Time window state (instead of zoom/pan in pixel space)
const [viewStartTs, setViewStartTs] = useState<number>(
traceMetadata.startTime,
);
const [viewEndTs, setViewEndTs] = useState<number>(traceMetadata.endTime);
const [scrollTop, setScrollTop] = useState<number>(0);
const [hoveredSpanId, setHoveredSpanId] = useState<string>('');
const renderSpanLevel = useCallback(
(_: number, spans: FlamegraphSpan[]): JSX.Element => (
@@ -157,16 +196,305 @@ function Success(props: ISuccessProps): JSX.Element {
});
}, [firstSpanAtFetchLevel, spans]);
// Draw a single event dot
const drawEventDot = useCallback(
(
ctx: CanvasRenderingContext2D,
x: number,
y: number,
isError: boolean,
): void => {
// could be optimized:
// ctx.beginPath();
// ctx.moveTo(x, y - size/2);
// ctx.lineTo(x + size/2, y);
// ctx.lineTo(x, y + size/2);
// ctx.lineTo(x - size/2, y);
// ctx.closePath();
// ctx.fill();
// ctx.stroke();
ctx.save();
ctx.translate(x, y);
ctx.rotate(Math.PI / 4); // 45 degrees
if (isError) {
ctx.fillStyle = isDarkMode ? 'rgb(239, 68, 68)' : 'rgb(220, 38, 38)';
ctx.strokeStyle = isDarkMode ? 'rgb(185, 28, 28)' : 'rgb(153, 27, 27)';
} else {
ctx.fillStyle = isDarkMode ? 'rgb(14, 165, 233)' : 'rgb(6, 182, 212)';
ctx.strokeStyle = isDarkMode ? 'rgb(2, 132, 199)' : 'rgb(8, 145, 178)';
}
ctx.lineWidth = 1;
ctx.fillRect(
-EVENT_DOT_SIZE / 2,
-EVENT_DOT_SIZE / 2,
EVENT_DOT_SIZE,
EVENT_DOT_SIZE,
);
ctx.strokeRect(
-EVENT_DOT_SIZE / 2,
-EVENT_DOT_SIZE / 2,
EVENT_DOT_SIZE,
EVENT_DOT_SIZE,
);
ctx.restore();
},
[isDarkMode],
);
// Get CSS color value from color string or CSS variable
// const getColorValue = useCallback((color: string): string => {
// // if (color.startsWith('var(')) {
// // // For CSS variables, we need to get computed value
// // const tempDiv = document.createElement('div');
// // tempDiv.style.color = color;
// // document.body.appendChild(tempDiv);
// // const computedColor = window.getComputedStyle(tempDiv).color;
// // document.body.removeChild(tempDiv);
// // return computedColor;
// // }
// return color;
// }, []);
// Get span color based on service, error state, and selection
// separate this when introducing interaction canvas
const getSpanColor = useCallback(
(span: FlamegraphSpan): string => {
let color = generateColor(span.serviceName, themeColors.traceDetailColors);
if (span.hasError) {
color = isDarkMode ? 'rgb(239, 68, 68)' : 'rgb(220, 38, 38)';
}
// else {
// color = getColorValue(color);
// }
// Apply selection/hover highlight
//hover/selection highlight in getSpanColor forces base redraw. clipping necessary.
if (selectedSpan?.spanId === span.spanId || hoveredSpanId === span.spanId) {
const colorObj = Color(color);
color = isDarkMode
? colorObj.lighten(0.7).hex()
: colorObj.darken(0.7).hex();
}
return color;
},
[isDarkMode, selectedSpan, hoveredSpanId],
);
// Draw a single span and its events
const drawSpan = useCallback(
(
ctx: CanvasRenderingContext2D,
span: FlamegraphSpan,
x: number,
y: number,
width: number,
levelIndex: number,
spanRectsArray: SpanRect[],
): void => {
const color = getSpanColor(span); // do not depend on hover/clicks
const spanY = y + SPAN_BAR_Y_OFFSET;
// Draw span rectangle
ctx.fillStyle = color;
ctx.beginPath();
// see if we can avoid roundRect as it is performance intensive
ctx.roundRect(x, spanY, width, SPAN_BAR_HEIGHT, 6);
ctx.fill();
// Store rect for hit testing
// consider per level buckets to improve hit testing
// So hover can:
// compute level from y
// search only within that row
spanRectsArray.push({
span,
x,
y: spanY,
width,
height: SPAN_BAR_HEIGHT,
level: levelIndex,
});
// Draw events
// think about optimizing this.
// if span is too small to draw events, skip drawing events???
span.event?.forEach((event) => {
const eventTimeMs = event.timeUnixNano / 1e6;
const eventOffsetPercent =
((eventTimeMs - span.timestamp) / (span.durationNano / 1e6)) * 100;
const clampedOffset = Math.max(1, Math.min(eventOffsetPercent, 99));
const eventX = x + (clampedOffset / 100) * width;
const eventY = spanY + SPAN_BAR_HEIGHT / 2;
// LOD guard: skip events if span too narrow
// if (width < EVENT_DOT_SIZE) {
// return;
// }
drawEventDot(ctx, eventX, eventY, event.isError);
});
},
[getSpanColor, drawEventDot],
);
const drawFlamegraph = useCallback(() => {
const canvas = baseCanvasRef.current;
const container = containerRef.current;
if (!canvas || !container) {
return;
}
const ctx = canvas.getContext('2d');
if (!ctx) {
return;
}
const dpr = window.devicePixelRatio || 1;
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
const timeSpan = viewEndTs - viewStartTs;
if (timeSpan <= 0) {
return;
}
const cssWidth = canvas.width / dpr;
// ---- Vertical clipping window ----
const viewportHeight = container.clientHeight;
const overscan = 4;
const firstLevel = Math.max(0, Math.floor(scrollTop / ROW_HEIGHT) - overscan);
const visibleLevelCount =
Math.ceil(viewportHeight / ROW_HEIGHT) + 2 * overscan;
const lastLevel = Math.min(spans.length - 1, firstLevel + visibleLevelCount);
// ---- Clear only visible region (recommended) ----
const clearTop = firstLevel * ROW_HEIGHT;
const clearHeight = (lastLevel - firstLevel + 1) * ROW_HEIGHT;
ctx.clearRect(0, clearTop, cssWidth, clearHeight);
const spanRectsArray: SpanRect[] = [];
// ---- Draw only visible levels ----
for (let levelIndex = firstLevel; levelIndex <= lastLevel; levelIndex++) {
const levelSpans = spans[levelIndex];
if (!levelSpans) {
continue;
}
const y = levelIndex * ROW_HEIGHT;
for (let i = 0; i < levelSpans.length; i++) {
const span = levelSpans[i];
const spanStartMs = span.timestamp;
const spanEndMs = span.timestamp + span.durationNano / 1e6;
// Time culling (already correct)
if (spanEndMs < viewStartTs || spanStartMs > viewEndTs) {
continue;
}
const leftOffset = ((spanStartMs - viewStartTs) / timeSpan) * cssWidth;
const rightEdge = ((spanEndMs - viewStartTs) / timeSpan) * cssWidth;
let width = rightEdge - leftOffset;
// Clamp to visible x-range
if (leftOffset < 0) {
width += leftOffset;
if (width <= 0) {
continue;
}
}
if (rightEdge > cssWidth) {
width = cssWidth - Math.max(0, leftOffset);
if (width <= 0) {
continue;
}
}
// Optional: minimum 1px width
if (width > 0 && width < 1) {
width = 1;
}
drawSpan(
ctx,
span,
Math.max(0, leftOffset),
y,
width,
levelIndex,
spanRectsArray,
);
}
}
spanRects.current = spanRectsArray;
}, [spans, viewStartTs, viewEndTs, scrollTop, drawSpan]);
// Handle canvas resize with device pixel ratio
useEffect(() => {
const canvas = baseCanvasRef.current;
const container = containerRef.current;
if (!canvas || !container) {
return;
}
const updateCanvasSize = (): void => {
const rect = container.getBoundingClientRect();
const dpr = window.devicePixelRatio || 1;
// Set CSS size
canvas.style.width = `${rect.width}px`;
canvas.style.height = `${totalHeight}px`;
// Set actual pixel size (accounting for DPR)
// Only update if size actually changed to prevent unnecessary redraws
const newWidth = Math.floor(rect.width * dpr);
const newHeight = Math.floor(totalHeight * dpr);
if (canvas.width !== newWidth || canvas.height !== newHeight) {
canvas.width = newWidth;
canvas.height = newHeight;
// Redraw with current time window (preserves zoom/pan)
drawFlamegraph();
}
};
const resizeObserver = new ResizeObserver(updateCanvasSize);
resizeObserver.observe(container);
// Initial size
updateCanvasSize();
// Handle DPR changes (e.g., moving window between screens)
const handleDPRChange = (): void => {
updateCanvasSize();
};
window
.matchMedia('(resolution: 1dppx)')
.addEventListener('change', handleDPRChange);
return (): void => {
resizeObserver.disconnect();
window
.matchMedia('(resolution: 1dppx)')
.removeEventListener('change', handleDPRChange);
};
}, [drawFlamegraph, totalHeight]);
// Re-draw when data changes
useEffect(() => {
drawFlamegraph();
}, [drawFlamegraph]);
return (
<>
<div className="trace-flamegraph">
<Virtuoso
ref={virtuosoRef}
className="trace-flamegraph-virtuoso"
data={spans}
itemContent={renderSpanLevel}
rangeChanged={handleRangeChanged}
/>
<div ref={containerRef} className="trace-flamegraph trace-flamegraph-canvas">
<canvas ref={baseCanvasRef}></canvas>
</div>
<TimelineV2
startTimestamp={traceMetadata.startTime}

View File

@@ -0,0 +1,890 @@
// @ts-nocheck
/* eslint-disable */
/* eslint-disable jsx-a11y/no-static-element-interactions */
/* eslint-disable jsx-a11y/click-events-have-key-events */
import {
Dispatch,
SetStateAction,
useCallback,
useEffect,
useRef,
useState,
} from 'react';
import { useHistory, useLocation } from 'react-router-dom';
import { Button } from 'antd';
import Color from 'color';
import TimelineV2 from 'components/TimelineV2/TimelineV2';
import { themeColors } from 'constants/theme';
import { useIsDarkMode } from 'hooks/useDarkMode';
import { generateColor } from 'lib/uPlotLib/utils/generateColor';
import { FlamegraphSpan } from 'types/api/trace/getTraceFlamegraph';
import { Span } from 'types/api/trace/getTraceV2';
import './Success.styles.scss';
interface ITraceMetadata {
startTime: number;
endTime: number;
}
interface ISuccessProps {
spans: FlamegraphSpan[][];
firstSpanAtFetchLevel: string;
setFirstSpanAtFetchLevel: Dispatch<SetStateAction<string>>;
traceMetadata: ITraceMetadata;
selectedSpan: Span | undefined;
}
// Constants for rendering
const ROW_HEIGHT = 24; // 18px height + 6px padding
const SPAN_BAR_HEIGHT = 12;
const EVENT_DOT_SIZE = 6;
const SPAN_BAR_Y_OFFSET = 3; // Center the 12px bar in 18px row
interface SpanRect {
span: FlamegraphSpan;
x: number;
y: number;
width: number;
height: number;
level: number;
}
function Success(props: ISuccessProps): JSX.Element {
const {
spans,
setFirstSpanAtFetchLevel,
traceMetadata,
firstSpanAtFetchLevel,
selectedSpan,
} = props;
const { search } = useLocation();
const history = useHistory();
const isDarkMode = useIsDarkMode();
const canvasRef = useRef<HTMLCanvasElement>(null);
const containerRef = useRef<HTMLDivElement>(null);
const [hoveredSpanId, setHoveredSpanId] = useState<string>('');
const [tooltipContent, setTooltipContent] = useState<{
content: string;
x: number;
y: number;
} | null>(null);
const [scrollTop, setScrollTop] = useState<number>(0);
// Time window state (instead of zoom/pan in pixel space)
const [viewStartTs, setViewStartTs] = useState<number>(
traceMetadata.startTime,
);
const [viewEndTs, setViewEndTs] = useState<number>(traceMetadata.endTime);
const [isSpacePressed, setIsSpacePressed] = useState<boolean>(false);
// Refs to avoid stale state during rapid wheel events and dragging
const viewStartRef = useRef(viewStartTs);
const viewEndRef = useRef(viewEndTs);
useEffect(() => {
viewStartRef.current = viewStartTs;
viewEndRef.current = viewEndTs;
}, [viewStartTs, viewEndTs]);
// Drag state in refs to avoid re-renders during drag
const isDraggingRef = useRef(false);
const dragStartRef = useRef<{ x: number; y: number } | null>(null);
const dragDistanceRef = useRef(0);
const suppressClickRef = useRef(false);
// Scroll ref to avoid recreating getCanvasPointer on every scroll
const scrollTopRef = useRef(0);
useEffect(() => {
scrollTopRef.current = scrollTop;
}, [scrollTop]);
// Build a flat array of span rectangles for hit testing
const spanRects = useRef<SpanRect[]>([]);
// Get span color based on service, error state, and selection
const getSpanColor = useCallback(
(span: FlamegraphSpan): string => {
let color = generateColor(span.serviceName, themeColors.traceDetailColors);
if (span.hasError) {
color = isDarkMode ? 'rgb(239, 68, 68)' : 'rgb(220, 38, 38)';
}
// Apply selection/hover highlight
if (selectedSpan?.spanId === span.spanId || hoveredSpanId === span.spanId) {
const colorObj = Color(color);
color = isDarkMode
? colorObj.lighten(0.7).hex()
: colorObj.darken(0.7).hex();
}
return color;
},
[isDarkMode, selectedSpan, hoveredSpanId],
);
// Draw a single event dot
const drawEventDot = useCallback(
(
ctx: CanvasRenderingContext2D,
x: number,
y: number,
isError: boolean,
): void => {
// could be optimized:
// ctx.beginPath();
// ctx.moveTo(x, y - size/2);
// ctx.lineTo(x + size/2, y);
// ctx.lineTo(x, y + size/2);
// ctx.lineTo(x - size/2, y);
// ctx.closePath();
// ctx.fill();
// ctx.stroke();
ctx.save();
ctx.translate(x, y);
ctx.rotate(Math.PI / 4); // 45 degrees
if (isError) {
ctx.fillStyle = isDarkMode ? 'rgb(239, 68, 68)' : 'rgb(220, 38, 38)';
ctx.strokeStyle = isDarkMode ? 'rgb(185, 28, 28)' : 'rgb(153, 27, 27)';
} else {
ctx.fillStyle = isDarkMode ? 'rgb(14, 165, 233)' : 'rgb(6, 182, 212)';
ctx.strokeStyle = isDarkMode ? 'rgb(2, 132, 199)' : 'rgb(8, 145, 178)';
}
ctx.lineWidth = 1;
ctx.fillRect(
-EVENT_DOT_SIZE / 2,
-EVENT_DOT_SIZE / 2,
EVENT_DOT_SIZE,
EVENT_DOT_SIZE,
);
ctx.strokeRect(
-EVENT_DOT_SIZE / 2,
-EVENT_DOT_SIZE / 2,
EVENT_DOT_SIZE,
EVENT_DOT_SIZE,
);
ctx.restore();
},
[isDarkMode],
);
// Draw a single span and its events
const drawSpan = useCallback(
(
ctx: CanvasRenderingContext2D,
span: FlamegraphSpan,
x: number,
y: number,
width: number,
levelIndex: number,
spanRectsArray: SpanRect[],
): void => {
const color = getSpanColor(span); // do not depend on hover/clicks
const spanY = y + SPAN_BAR_Y_OFFSET;
// Draw span rectangle
ctx.fillStyle = color;
ctx.beginPath();
ctx.roundRect(x, spanY, width, SPAN_BAR_HEIGHT, 6);
ctx.fill();
// Store rect for hit testing
// consider per level buckets to improve hit testing
// So hover can:
// compute level from y
// search only within that row
spanRectsArray.push({
span,
x,
y: spanY,
width,
height: SPAN_BAR_HEIGHT,
level: levelIndex,
});
// Draw events
// think about optimizing this.
// if span is too small to draw events, skip drawing events???
span.event?.forEach((event) => {
const eventTimeMs = event.timeUnixNano / 1e6;
const eventOffsetPercent =
((eventTimeMs - span.timestamp) / (span.durationNano / 1e6)) * 100;
const clampedOffset = Math.max(1, Math.min(eventOffsetPercent, 99));
const eventX = x + (clampedOffset / 100) * width;
const eventY = spanY + SPAN_BAR_HEIGHT / 2;
drawEventDot(ctx, eventX, eventY, event.isError);
});
},
[getSpanColor, drawEventDot],
);
// Draw the flamegraph on canvas
const drawFlamegraph = useCallback(() => {
const canvas = canvasRef.current;
const container = containerRef.current;
if (!canvas || !container) {
return;
}
const ctx = canvas.getContext('2d');
if (!ctx) {
return;
}
const dpr = window.devicePixelRatio || 1;
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
const timeSpan = viewEndTs - viewStartTs;
if (timeSpan <= 0) {
return;
}
const cssWidth = canvas.width / dpr;
// ---- Vertical clipping window ----
const viewportHeight = container.clientHeight;
const overscan = 4;
const firstLevel = Math.max(0, Math.floor(scrollTop / ROW_HEIGHT) - overscan);
const visibleLevelCount =
Math.ceil(viewportHeight / ROW_HEIGHT) + 2 * overscan;
const lastLevel = Math.min(spans.length - 1, firstLevel + visibleLevelCount);
// ---- Clear only visible region (recommended) ----
const clearTop = firstLevel * ROW_HEIGHT;
const clearHeight = (lastLevel - firstLevel + 1) * ROW_HEIGHT;
ctx.clearRect(0, clearTop, cssWidth, clearHeight);
const spanRectsArray: SpanRect[] = [];
// ---- Draw only visible levels ----
for (let levelIndex = firstLevel; levelIndex <= lastLevel; levelIndex++) {
const levelSpans = spans[levelIndex];
if (!levelSpans) {
continue;
}
const y = levelIndex * ROW_HEIGHT;
for (let i = 0; i < levelSpans.length; i++) {
const span = levelSpans[i];
const spanStartMs = span.timestamp;
const spanEndMs = span.timestamp + span.durationNano / 1e6;
// Time culling (already correct)
if (spanEndMs < viewStartTs || spanStartMs > viewEndTs) {
continue;
}
const leftOffset = ((spanStartMs - viewStartTs) / timeSpan) * cssWidth;
const rightEdge = ((spanEndMs - viewStartTs) / timeSpan) * cssWidth;
let width = rightEdge - leftOffset;
// Clamp to visible x-range
if (leftOffset < 0) {
width += leftOffset;
if (width <= 0) {
continue;
}
}
if (rightEdge > cssWidth) {
width = cssWidth - Math.max(0, leftOffset);
if (width <= 0) {
continue;
}
}
// Optional: minimum 1px width
if (width > 0 && width < 1) {
width = 1;
}
drawSpan(
ctx,
span,
Math.max(0, leftOffset),
y,
width,
levelIndex,
spanRectsArray,
);
}
}
spanRects.current = spanRectsArray;
}, [spans, viewStartTs, viewEndTs, scrollTop, drawSpan]);
// Calculate total canvas height
const totalHeight = spans.length * ROW_HEIGHT;
console.log('time: ', {
start: traceMetadata.startTime,
end: traceMetadata.endTime,
});
// Initialize time window when trace metadata changes (only if not already set)
useEffect(() => {
// Only reset if we're at the default view (full trace)
// This prevents resetting zoom/pan when metadata updates
if (
viewStartTs === traceMetadata.startTime &&
viewEndTs === traceMetadata.endTime
) {
// Already at default, no need to update
return;
}
// Only reset if the trace bounds have actually changed significantly
const currentSpan = viewEndTs - viewStartTs;
const fullSpan = traceMetadata.endTime - traceMetadata.startTime;
// If we're zoomed in, preserve the zoom level relative to new bounds
if (currentSpan < fullSpan * 0.99) {
// We're zoomed in, adjust the window proportionally
const ratio = currentSpan / fullSpan;
const newSpan = (traceMetadata.endTime - traceMetadata.startTime) * ratio;
const center = (viewStartTs + viewEndTs) / 2;
const newStart = Math.max(
traceMetadata.startTime,
Math.min(center - newSpan / 2, traceMetadata.endTime - newSpan),
);
setViewStartTs(newStart);
setViewEndTs(newStart + newSpan);
} else {
// We're at full view, reset to new full view
setViewStartTs(traceMetadata.startTime);
setViewEndTs(traceMetadata.endTime);
}
}, [traceMetadata.startTime, traceMetadata.endTime, viewStartTs, viewEndTs]);
// Handle canvas resize with device pixel ratio
useEffect(() => {
const canvas = canvasRef.current;
const container = containerRef.current;
if (!canvas || !container) {
return;
}
const updateCanvasSize = (): void => {
const rect = container.getBoundingClientRect();
const dpr = window.devicePixelRatio || 1;
// Set CSS size
canvas.style.width = `${rect.width}px`;
canvas.style.height = `${totalHeight}px`;
// Set actual pixel size (accounting for DPR)
// Only update if size actually changed to prevent unnecessary redraws
const newWidth = Math.floor(rect.width * dpr);
const newHeight = Math.floor(totalHeight * dpr);
if (canvas.width !== newWidth || canvas.height !== newHeight) {
canvas.width = newWidth;
canvas.height = newHeight;
// Redraw with current time window (preserves zoom/pan)
drawFlamegraph();
}
};
const resizeObserver = new ResizeObserver(updateCanvasSize);
resizeObserver.observe(container);
// Initial size
updateCanvasSize();
// Handle DPR changes (e.g., moving window between screens)
const handleDPRChange = (): void => {
updateCanvasSize();
};
window
.matchMedia('(resolution: 1dppx)')
.addEventListener('change', handleDPRChange);
return (): void => {
resizeObserver.disconnect();
window
.matchMedia('(resolution: 1dppx)')
.removeEventListener('change', handleDPRChange);
};
}, [drawFlamegraph, totalHeight]);
// Re-draw when data changes
useEffect(() => {
drawFlamegraph();
}, [drawFlamegraph]);
// Find span at given canvas coordinates
const findSpanAtPosition = useCallback((x: number, y: number):
| SpanRect
| undefined => {
return spanRects.current.find(
(spanRect) =>
x >= spanRect.x &&
x <= spanRect.x + spanRect.width &&
y >= spanRect.y &&
y <= spanRect.y + spanRect.height,
);
}, []);
// Utility to convert client coordinates to CSS canvas coordinates
const getCanvasPointer = useCallback((clientX: number, clientY: number): {
cssX: number;
cssY: number;
cssWidth: number;
} | null => {
const canvas = canvasRef.current;
if (!canvas) {
return null;
}
const rect = canvas.getBoundingClientRect();
const dpr = window.devicePixelRatio || 1;
const cssWidth = canvas.width / dpr;
const cssX = (clientX - rect.left) * (cssWidth / rect.width);
const cssY = clientY - rect.top + scrollTopRef.current;
return { cssX, cssY, cssWidth };
}, []);
// Handle mouse move for hover and dragging
const handleMouseMove = useCallback(
(event: React.MouseEvent<HTMLCanvasElement>) => {
const canvas = canvasRef.current;
if (!canvas) {
return;
}
const rect = canvas.getBoundingClientRect();
console.log('event', { clientX: event.clientX, clientY: event.clientY });
// ---- Dragging (pan in time space) ----
if (isDraggingRef.current && dragStartRef.current) {
const deltaX = event.clientX - dragStartRef.current.x;
const deltaY = event.clientY - dragStartRef.current.y;
console.log('delta', { deltaY, deltaX });
const totalDistance = Math.sqrt(deltaX * deltaX + deltaY * deltaY);
dragDistanceRef.current = totalDistance;
const timeSpan = viewEndRef.current - viewStartRef.current;
const deltaTime = (deltaX / rect.width) * timeSpan;
const newStart = viewStartRef.current - deltaTime;
const clampedStart = Math.max(
traceMetadata.startTime,
Math.min(newStart, traceMetadata.endTime - timeSpan),
);
const clampedEnd = clampedStart + timeSpan;
setViewStartTs(clampedStart);
setViewEndTs(clampedEnd);
dragStartRef.current = {
x: event.clientX,
y: event.clientY,
};
return;
}
// ---- Hover ----
const pointer = getCanvasPointer(event.clientX, event.clientY);
if (!pointer) {
return;
}
const { cssX, cssY } = pointer;
const hoveredSpan = findSpanAtPosition(cssX, cssY);
if (hoveredSpan) {
setHoveredSpanId(hoveredSpan.span.spanId);
setTooltipContent({
content: hoveredSpan.span.name,
x: event.clientX,
y: event.clientY,
});
canvas.style.cursor = 'pointer';
} else {
setHoveredSpanId('');
setTooltipContent(null);
// Set cursor based on space key state when not hovering
canvas.style.cursor = isSpacePressed ? 'grab' : 'default';
}
},
[findSpanAtPosition, traceMetadata, getCanvasPointer, isSpacePressed],
);
// Handle key down for space key
useEffect(() => {
const handleKeyDown = (event: KeyboardEvent): void => {
if (event.code === 'Space') {
event.preventDefault();
setIsSpacePressed(true);
}
};
const handleKeyUp = (event: KeyboardEvent): void => {
if (event.code === 'Space') {
event.preventDefault();
setIsSpacePressed(false);
}
};
window.addEventListener('keydown', handleKeyDown);
window.addEventListener('keyup', handleKeyUp);
return (): void => {
window.removeEventListener('keydown', handleKeyDown);
window.removeEventListener('keyup', handleKeyUp);
};
}, []);
const handleMouseDown = useCallback(
(event: React.MouseEvent<HTMLCanvasElement>) => {
if (event.button !== 0) {
return;
} // left click only
event.preventDefault();
isDraggingRef.current = true;
dragStartRef.current = {
x: event.clientX,
y: event.clientY,
};
dragDistanceRef.current = 0;
const canvas = canvasRef.current;
if (canvas) {
canvas.style.cursor = 'grabbing';
}
},
[],
);
const handleMouseUp = useCallback(() => {
const wasDrag = dragDistanceRef.current > 5;
suppressClickRef.current = wasDrag; // 👈 key fix: suppress click after drag
isDraggingRef.current = false;
dragStartRef.current = null;
dragDistanceRef.current = 0;
const canvas = canvasRef.current;
if (canvas) {
canvas.style.cursor = 'grab';
}
return wasDrag;
}, []);
const handleMouseLeave = useCallback(() => {
isOverFlamegraphRef.current = false;
setHoveredSpanId('');
setTooltipContent(null);
isDraggingRef.current = false;
dragStartRef.current = null;
dragDistanceRef.current = 0;
const canvas = canvasRef.current;
if (canvas) {
canvas.style.cursor = 'grab';
}
}, []);
const handleClick = useCallback(
(event: React.MouseEvent<HTMLCanvasElement>) => {
// Prevent click after drag
if (suppressClickRef.current) {
suppressClickRef.current = false; // reset after suppressing once
return;
}
const pointer = getCanvasPointer(event.clientX, event.clientY);
if (!pointer) {
return;
}
const { cssX, cssY } = pointer;
const clickedSpan = findSpanAtPosition(cssX, cssY);
if (!clickedSpan) {
return;
}
const searchParams = new URLSearchParams(search);
const currentSpanId = searchParams.get('spanId');
if (currentSpanId !== clickedSpan.span.spanId) {
searchParams.set('spanId', clickedSpan.span.spanId);
history.replace({ search: searchParams.toString() });
}
},
[search, history, findSpanAtPosition, getCanvasPointer],
);
const isOverFlamegraphRef = useRef(false);
useEffect(() => {
const onWheel = (e: WheelEvent) => {
// Pinch zoom on trackpads often comes as ctrl+wheel
if (isOverFlamegraphRef.current && e.ctrlKey) {
e.preventDefault(); // stops browser zoom
}
};
// capture:true ensures we intercept early
window.addEventListener('wheel', onWheel, { passive: false, capture: true });
return () => {
window.removeEventListener(
'wheel',
onWheel as any,
{ capture: true } as any,
);
};
}, []);
const wheelDeltaRef = useRef(0);
const rafRef = useRef<number | null>(null);
const lastCursorXRef = useRef(0);
const lastCssWidthRef = useRef(1);
const lastIsPinchRef = useRef(false);
const applyWheelZoom = useCallback(() => {
rafRef.current = null;
const cssWidth = lastCssWidthRef.current || 1;
const cursorX = lastCursorXRef.current;
const fullSpan = traceMetadata.endTime - traceMetadata.startTime;
const oldSpan = viewEndRef.current - viewStartRef.current;
// ✅ Different intensity for pinch vs scroll
const zoomIntensityScroll = 0.0015;
const zoomIntensityPinch = 0.01; // pinch needs stronger response
const zoomIntensity = lastIsPinchRef.current
? zoomIntensityPinch
: zoomIntensityScroll;
const deltaY = wheelDeltaRef.current;
wheelDeltaRef.current = 0;
// ✅ Smooth zoom using delta magnitude
const zoomFactor = Math.exp(deltaY * zoomIntensity);
const newSpan = oldSpan * zoomFactor;
console.log('newSpan', { cssWidth, newSpan, zoomFactor, oldSpan });
// ✅ Better minSpan clamp (absolute + pixel-based)
const absoluteMinSpan = 5; // ms
const pixelMinSpan = fullSpan / cssWidth; // ~1px of time
const minSpan = Math.max(absoluteMinSpan, pixelMinSpan);
const maxSpan = fullSpan;
const clampedSpan = Math.max(minSpan, Math.min(maxSpan, newSpan));
// ✅ Anchor preserving zoom (same as your original logic)
const cursorRatio = Math.max(0, Math.min(cursorX / cssWidth, 1));
const anchorTs = viewStartRef.current + cursorRatio * oldSpan;
const newViewStart = anchorTs - cursorRatio * clampedSpan;
const finalStart = Math.max(
traceMetadata.startTime,
Math.min(newViewStart, traceMetadata.endTime - clampedSpan),
);
const finalEnd = finalStart + clampedSpan;
console.log('finalStart', finalStart);
console.log('finalEnd', finalEnd);
setViewStartTs(finalStart);
setViewEndTs(finalEnd);
}, [traceMetadata]);
const handleWheel = useCallback(
(event: React.WheelEvent<HTMLCanvasElement>) => {
event.preventDefault();
const pointer = getCanvasPointer(event.clientX, event.clientY);
if (!pointer) {
return;
}
console.log('pointer', pointer);
const { cssX: cursorX, cssWidth } = pointer;
// ✅ Detect pinch on Chrome/Edge: ctrlKey true for trackpad pinch
lastIsPinchRef.current = event.ctrlKey;
lastCssWidthRef.current = cssWidth;
lastCursorXRef.current = cursorX;
// ✅ Accumulate deltas; apply once per frame
wheelDeltaRef.current += event.deltaY;
if (rafRef.current == null) {
rafRef.current = requestAnimationFrame(applyWheelZoom);
}
},
[applyWheelZoom, getCanvasPointer],
);
// Reset zoom and pan
const handleResetZoom = useCallback(() => {
setViewStartTs(traceMetadata.startTime);
setViewEndTs(traceMetadata.endTime);
}, [traceMetadata]);
// Handle scroll for pagination
const handleScroll = useCallback(
(event: React.UIEvent<HTMLDivElement>): void => {
const target = event.currentTarget;
setScrollTop(target.scrollTop);
// Pagination logic
if (spans.length < 50) {
return;
}
const scrollPercentage = target.scrollTop / target.scrollHeight;
const totalLevels = spans.length;
if (scrollPercentage === 0 && spans[0]?.[0]?.level !== 0) {
setFirstSpanAtFetchLevel(spans[0][0].spanId);
}
if (scrollPercentage >= 0.95 && spans[totalLevels - 1]?.[0]?.spanId) {
setFirstSpanAtFetchLevel(spans[totalLevels - 1][0].spanId);
}
},
[spans, setFirstSpanAtFetchLevel],
);
// Auto-scroll to selected span
useEffect(() => {
if (!firstSpanAtFetchLevel || !containerRef.current) {
return;
}
const levelIndex = spans.findIndex(
(level) => level[0]?.spanId === firstSpanAtFetchLevel,
);
if (levelIndex !== -1) {
const targetScroll = levelIndex * ROW_HEIGHT;
containerRef.current.scrollTop = targetScroll;
setScrollTop(targetScroll);
}
}, [firstSpanAtFetchLevel, spans]);
return (
<>
<div
ref={containerRef}
className="trace-flamegraph trace-flamegraph-canvas"
onScroll={handleScroll}
>
{(viewStartTs !== traceMetadata.startTime ||
viewEndTs !== traceMetadata.endTime) && (
<Button
className="flamegraph-reset-zoom"
size="small"
onClick={handleResetZoom}
title="Reset zoom and pan"
>
Reset View
</Button>
)}
<canvas
ref={canvasRef}
style={{
display: 'block',
width: '100%',
height: `${totalHeight}px`,
}}
onMouseMove={handleMouseMove}
onMouseDown={handleMouseDown}
onMouseUp={handleMouseUp}
onMouseEnter={() => (isOverFlamegraphRef.current = true)}
onMouseLeave={handleMouseLeave}
onClick={handleClick}
onWheel={handleWheel}
onContextMenu={(e): void => e.preventDefault()}
/>
{tooltipContent && (
<div
style={{
position: 'fixed',
left: `${tooltipContent.x + 10}px`,
top: `${tooltipContent.y - 10}px`,
pointerEvents: 'none',
zIndex: 1000,
backgroundColor: isDarkMode ? '#1f2937' : '#ffffff',
color: isDarkMode ? '#ffffff' : '#000000',
padding: '4px 8px',
borderRadius: '4px',
fontSize: '12px',
boxShadow: '0 2px 8px rgba(0,0,0,0.15)',
}}
>
{tooltipContent.content}
</div>
)}
</div>
<TimelineV2
startTimestamp={viewStartTs}
endTimestamp={viewEndTs}
offsetTimestamp={viewStartTs - traceMetadata.startTime}
timelineHeight={22}
/>
</>
);
}
export default Success;
// on drag on click is getting registered as a click
// zoom and scale not matching
// check minimap logic
// use
// const scrollTopRef = useRef(scrollTop);
// useEffect(() => {
// scrollTopRef.current = scrollTop;
// }, [scrollTop]);
// fix clicks in interaction canvas
// Auto-scroll to selected span else on top(based on default span)
// time bar line vertical
// zoom handle vertical and horizontal scroll with proper defined thresholds
// timeline should be in sync with the flamegraph. test with vertical line of time on event etc.
// proper working interaction layer for clicks and hovers
// hit testing should be efficient and accurate without flat spanRect
// Final Priority Order (Clean Summary)
// ✅ Zoom (Horizontal + Vertical thresholds)
// ✅ Timeline sync + vertical time dashed line
// ✅ Minimap brush correctness
// ✅ Auto-scroll behavior
// ✅ Interaction layer separation
// ✅ Efficient hit testing

View File

@@ -17,6 +17,9 @@ const useGetTraceFlamegraph = (
REACT_QUERY_KEY.GET_TRACE_V2_FLAMEGRAPH,
props.traceId,
props.selectedSpanId,
props.limit,
props.boundaryStartTsMilli,
props.boundarEndTsMilli,
],
enabled: !!props.traceId,
keepPreviousData: true,

View File

@@ -5,6 +5,9 @@ export interface TraceDetailFlamegraphURLProps {
export interface GetTraceFlamegraphPayloadProps {
traceId: string;
selectedSpanId: string;
limit: number;
boundaryStartTsMilli: number;
boundarEndTsMilli: number;
}
export interface Event {
@@ -31,4 +34,6 @@ export interface GetTraceFlamegraphSuccessResponse {
spans: FlamegraphSpan[][];
startTimestampMillis: number;
endTimestampMillis: number;
durationNano: number;
hasMore: boolean;
}

View File

@@ -18,17 +18,13 @@ type responseerroradditional struct {
func AsJSON(cause error) *JSON {
// See if this is an instance of the base error or not
_, c, m, err, u, a := Unwrapb(cause)
_, c, m, _, u, a := Unwrapb(cause)
rea := make([]responseerroradditional, len(a))
for k, v := range a {
rea[k] = responseerroradditional{v}
}
if err != nil {
rea = append(rea, responseerroradditional{err.Error()})
}
return &JSON{
Code: c.String(),
Message: m,

View File

@@ -120,6 +120,8 @@ func FilterResponse(results []*qbtypes.QueryRangeResponse) []*qbtypes.QueryRange
}
}
resultData.Rows = filteredRows
case *qbtypes.ScalarData:
resultData.Data = filterScalarDataIPs(resultData.Columns, resultData.Data)
}
filteredData = append(filteredData, result)
@@ -145,6 +147,39 @@ func shouldIncludeSeries(series *qbtypes.TimeSeries) bool {
return true
}
func filterScalarDataIPs(columns []*qbtypes.ColumnDescriptor, data [][]any) [][]any {
// Find column indices for server address fields
serverColIndices := make([]int, 0)
for i, col := range columns {
if col.Name == derivedKeyHTTPHost {
serverColIndices = append(serverColIndices, i)
}
}
if len(serverColIndices) == 0 {
return data
}
filtered := make([][]any, 0, len(data))
for _, row := range data {
includeRow := true
for _, colIdx := range serverColIndices {
if colIdx < len(row) {
if strVal, ok := row[colIdx].(string); ok {
if net.ParseIP(strVal) != nil {
includeRow = false
break
}
}
}
}
if includeRow {
filtered = append(filtered, row)
}
}
return filtered
}
func shouldIncludeRow(row *qbtypes.RawRow) bool {
if row.Data != nil {
if domainVal, ok := row.Data[derivedKeyHTTPHost]; ok {

View File

@@ -117,6 +117,59 @@ func TestFilterResponse(t *testing.T) {
},
},
},
{
name: "should filter out IP addresses from scalar data",
input: []*qbtypes.QueryRangeResponse{
{
Data: qbtypes.QueryData{
Results: []any{
&qbtypes.ScalarData{
QueryName: "endpoints",
Columns: []*qbtypes.ColumnDescriptor{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: derivedKeyHTTPHost},
Type: qbtypes.ColumnTypeGroup,
},
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "endpoints"},
Type: qbtypes.ColumnTypeAggregation,
},
},
Data: [][]any{
{"192.168.1.1", 10},
{"example.com", 20},
{"10.0.0.1", 5},
},
},
},
},
},
},
expected: []*qbtypes.QueryRangeResponse{
{
Data: qbtypes.QueryData{
Results: []any{
&qbtypes.ScalarData{
QueryName: "endpoints",
Columns: []*qbtypes.ColumnDescriptor{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: derivedKeyHTTPHost},
Type: qbtypes.ColumnTypeGroup,
},
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "endpoints"},
Type: qbtypes.ColumnTypeAggregation,
},
},
Data: [][]any{
{"example.com", 20},
},
},
},
},
},
},
},
}
for _, tt := range tests {

View File

@@ -1046,7 +1046,19 @@ func (r *ClickHouseReader) GetWaterfallSpansForTraceWithMetadata(ctx context.Con
}
processingPostCache := time.Now()
selectedSpans, uncollapsedSpans, rootServiceName, rootServiceEntryPoint := tracedetail.GetSelectedSpans(req.UncollapsedSpans, req.SelectedSpanID, traceRoots, spanIdToSpanNodeMap, req.IsSelectedSpanIDUnCollapsed)
limit := min(req.Limit, tracedetail.MaxLimitToSelectAllSpans)
selectAllSpans := totalSpans <= uint64(limit)
var (
selectedSpans []*model.Span
uncollapsedSpans []string
rootServiceName, rootServiceEntryPoint string
)
if selectAllSpans {
selectedSpans, rootServiceName, rootServiceEntryPoint = tracedetail.GetAllSpans(traceRoots)
} else {
selectedSpans, uncollapsedSpans, rootServiceName, rootServiceEntryPoint = tracedetail.GetSelectedSpans(req.UncollapsedSpans, req.SelectedSpanID, traceRoots, spanIdToSpanNodeMap, req.IsSelectedSpanIDUnCollapsed)
}
zap.L().Info("getWaterfallSpansForTraceWithMetadata: processing post cache", zap.Duration("duration", time.Since(processingPostCache)), zap.String("traceID", traceID))
// convert start timestamp to millis because right now frontend is expecting it in millis
@@ -1059,7 +1071,7 @@ func (r *ClickHouseReader) GetWaterfallSpansForTraceWithMetadata(ctx context.Con
}
response.Spans = selectedSpans
response.UncollapsedSpans = uncollapsedSpans
response.UncollapsedSpans = uncollapsedSpans // ignoring if all spans are returning
response.StartTimestampMillis = startTime / 1000000
response.EndTimestampMillis = endTime / 1000000
response.TotalSpansCount = totalSpans
@@ -1068,6 +1080,7 @@ func (r *ClickHouseReader) GetWaterfallSpansForTraceWithMetadata(ctx context.Con
response.RootServiceEntryPoint = rootServiceEntryPoint
response.ServiceNameToTotalDurationMap = serviceNameToTotalDurationMap
response.HasMissingSpans = hasMissingSpans
response.HasMore = !selectAllSpans
return response, nil
}
@@ -1199,7 +1212,7 @@ func (r *ClickHouseReader) GetFlamegraphSpansForTrace(ctx context.Context, orgID
}
}
selectedSpans = tracedetail.GetSelectedSpansForFlamegraph(traceRoots, spanIdToSpanNodeMap)
selectedSpans = tracedetail.GetAllSpansForFlamegraph(traceRoots, spanIdToSpanNodeMap)
traceCache := model.GetFlamegraphSpansForTraceCache{
StartTime: startTime,
EndTime: endTime,
@@ -1216,12 +1229,20 @@ func (r *ClickHouseReader) GetFlamegraphSpansForTrace(ctx context.Context, orgID
}
processingPostCache := time.Now()
selectedSpansForRequest := tracedetail.GetSelectedSpansForFlamegraphForRequest(req.SelectedSpanID, selectedSpans, startTime, endTime)
zap.L().Info("getFlamegraphSpansForTrace: processing post cache", zap.Duration("duration", time.Since(processingPostCache)), zap.String("traceID", traceID))
selectedSpansForRequest := selectedSpans
limit := min(req.Limit, tracedetail.MaxLimitWithoutSampling)
totalSpanCount := tracedetail.GetTotalSpanCount(selectedSpans)
if totalSpanCount > uint64(limit) {
boundaryStart, boundaryEnd := utils.MilliToNano(req.BoundaryStartTS), utils.MilliToNano(req.BoundaryEndTS)
selectedSpansForRequest = tracedetail.GetSelectedSpansForFlamegraphForRequest(req.SelectedSpanID, selectedSpans, boundaryStart, boundaryEnd)
}
zap.L().Info("getFlamegraphSpansForTrace: processing post cache", zap.Duration("duration", time.Since(processingPostCache)), zap.String("traceID", traceID),
zap.Uint64("totalSpanCount", totalSpanCount))
trace.Spans = selectedSpansForRequest
trace.StartTimestampMillis = startTime / 1000000
trace.EndTimestampMillis = endTime / 1000000
trace.HasMore = totalSpanCount > uint64(limit)
return trace, nil
}

View File

@@ -7,9 +7,11 @@ import (
)
var (
SPAN_LIMIT_PER_REQUEST_FOR_FLAMEGRAPH float64 = 50
SPAN_LIMIT_PER_LEVEL int = 100
TIMESTAMP_SAMPLING_BUCKET_COUNT int = 50
flamegraphSpanLevelLimit float64 = 50
flamegraphSpanLimitPerLevel int = 1000
flamegraphSamplingBucketCount int = 500
MaxLimitWithoutSampling uint = 120_000
)
func ContainsFlamegraphSpan(slice []*model.FlamegraphSpan, item *model.FlamegraphSpan) bool {
@@ -52,7 +54,8 @@ func FindIndexForSelectedSpan(spans [][]*model.FlamegraphSpan, selectedSpanId st
return selectedSpanLevel
}
func GetSelectedSpansForFlamegraph(traceRoots []*model.FlamegraphSpan, spanIdToSpanNodeMap map[string]*model.FlamegraphSpan) [][]*model.FlamegraphSpan {
// GetAllSpansForFlamegraph groups all spans as per their level
func GetAllSpansForFlamegraph(traceRoots []*model.FlamegraphSpan, spanIdToSpanNodeMap map[string]*model.FlamegraphSpan) [][]*model.FlamegraphSpan {
var traceIdLevelledFlamegraph = map[string]map[int64][]*model.FlamegraphSpan{}
selectedSpans := [][]*model.FlamegraphSpan{}
@@ -100,7 +103,7 @@ func getLatencyAndTimestampBucketedSpans(spans []*model.FlamegraphSpan, selected
})
// pick the top 5 latency spans
for idx := range 5 {
for idx := range 100 {
sampledSpans = append(sampledSpans, spans[idx])
}
@@ -110,6 +113,7 @@ func getLatencyAndTimestampBucketedSpans(spans []*model.FlamegraphSpan, selected
for _idx, span := range spans {
if span.SpanID == selectedSpanID {
idx = _idx
break
}
}
if idx != -1 {
@@ -117,17 +121,17 @@ func getLatencyAndTimestampBucketedSpans(spans []*model.FlamegraphSpan, selected
}
}
bucketSize := (endTime - startTime) / uint64(TIMESTAMP_SAMPLING_BUCKET_COUNT)
bucketSize := (endTime - startTime) / uint64(flamegraphSamplingBucketCount)
if bucketSize == 0 {
bucketSize = 1
}
bucketedSpans := make([][]*model.FlamegraphSpan, 50)
bucketedSpans := make([][]*model.FlamegraphSpan, flamegraphSamplingBucketCount)
for _, span := range spans {
if span.TimeUnixNano >= startTime && span.TimeUnixNano <= endTime {
bucketIndex := int((span.TimeUnixNano - startTime) / bucketSize)
if bucketIndex >= 0 && bucketIndex < 50 {
if bucketIndex >= 0 && bucketIndex < flamegraphSamplingBucketCount {
bucketedSpans[bucketIndex] = append(bucketedSpans[bucketIndex], span)
}
}
@@ -156,8 +160,8 @@ func GetSelectedSpansForFlamegraphForRequest(selectedSpanID string, selectedSpan
selectedIndex = FindIndexForSelectedSpan(selectedSpans, selectedSpanID)
}
lowerLimit := selectedIndex - int(SPAN_LIMIT_PER_REQUEST_FOR_FLAMEGRAPH*0.4)
upperLimit := selectedIndex + int(SPAN_LIMIT_PER_REQUEST_FOR_FLAMEGRAPH*0.6)
lowerLimit := selectedIndex - int(flamegraphSpanLevelLimit*0.4)
upperLimit := selectedIndex + int(flamegraphSpanLevelLimit*0.6)
if lowerLimit < 0 {
upperLimit = upperLimit - lowerLimit
@@ -174,7 +178,7 @@ func GetSelectedSpansForFlamegraphForRequest(selectedSpanID string, selectedSpan
}
for i := lowerLimit; i < upperLimit; i++ {
if len(selectedSpans[i]) > SPAN_LIMIT_PER_LEVEL {
if len(selectedSpans[i]) > flamegraphSpanLimitPerLevel {
_spans := getLatencyAndTimestampBucketedSpans(selectedSpans[i], selectedSpanID, i == selectedIndex, startTime, endTime)
selectedSpansForRequest = append(selectedSpansForRequest, _spans)
} else {
@@ -184,3 +188,12 @@ func GetSelectedSpansForFlamegraphForRequest(selectedSpanID string, selectedSpan
return selectedSpansForRequest
}
func GetTotalSpanCount(spans [][]*model.FlamegraphSpan) uint64 {
levelCount := len(spans)
spanCount := uint64(0)
for i := range levelCount {
spanCount += uint64(len(spans[i]))
}
return spanCount
}

View File

@@ -9,6 +9,9 @@ import (
var (
SPAN_LIMIT_PER_REQUEST_FOR_WATERFALL float64 = 500
maxDepthForSelectedSpanChildren int = 5
MaxLimitToSelectAllSpans uint = 10_000
)
type Interval struct {
@@ -88,8 +91,11 @@ func getPathFromRootToSelectedSpanId(node *model.Span, selectedSpanId string, un
return isPresentInSubtreeForTheNode, spansFromRootToNode
}
func traverseTrace(span *model.Span, uncollapsedSpans []string, level uint64, isPartOfPreOrder bool, hasSibling bool, selectedSpanId string) []*model.Span {
func traverseTrace(span *model.Span, uncollapsedSpans []string, level uint64, isPartOfPreOrder bool, hasSibling bool, selectedSpanId string,
depthFromSelectedSpan int, isSelectedSpanIDUnCollapsed bool, selectAllSpan bool) ([]*model.Span, []string) {
preOrderTraversal := []*model.Span{}
autoExpandedSpans := []string{}
// sort the children to maintain the order across requests
sort.Slice(span.Children, func(i, j int) bool {
@@ -126,15 +132,40 @@ func traverseTrace(span *model.Span, uncollapsedSpans []string, level uint64, is
preOrderTraversal = append(preOrderTraversal, &nodeWithoutChildren)
}
nextDepthFromSelectedSpan := -1
if span.SpanID == selectedSpanId && isSelectedSpanIDUnCollapsed {
nextDepthFromSelectedSpan = 1
} else if depthFromSelectedSpan >= 1 && depthFromSelectedSpan < maxDepthForSelectedSpanChildren {
nextDepthFromSelectedSpan = depthFromSelectedSpan + 1
}
for index, child := range span.Children {
_childTraversal := traverseTrace(child, uncollapsedSpans, level+1, isPartOfPreOrder && slices.Contains(uncollapsedSpans, span.SpanID), index != (len(span.Children)-1), selectedSpanId)
// A child is included in the pre-order output if its parent is uncollapsed
// OR if the child falls within MAX_DEPTH_FOR_SELECTED_SPAN_CHILDREN levels
// below the selected span.
isChildWithinMaxDepth := nextDepthFromSelectedSpan >= 1
isAlreadyUncollapsed := slices.Contains(uncollapsedSpans, span.SpanID)
childIsPartOfPreOrder := isPartOfPreOrder && (isAlreadyUncollapsed || isChildWithinMaxDepth)
if selectAllSpan {
childIsPartOfPreOrder = true
}
if isPartOfPreOrder && isChildWithinMaxDepth && !isAlreadyUncollapsed {
if !slices.Contains(autoExpandedSpans, span.SpanID) {
autoExpandedSpans = append(autoExpandedSpans, span.SpanID)
}
}
_childTraversal, _autoExpanded := traverseTrace(child, uncollapsedSpans, level+1, childIsPartOfPreOrder, index != (len(span.Children)-1), selectedSpanId,
nextDepthFromSelectedSpan, isSelectedSpanIDUnCollapsed, selectAllSpan)
preOrderTraversal = append(preOrderTraversal, _childTraversal...)
autoExpandedSpans = append(autoExpandedSpans, _autoExpanded...)
nodeWithoutChildren.SubTreeNodeCount += child.SubTreeNodeCount + 1
span.SubTreeNodeCount += child.SubTreeNodeCount + 1
}
nodeWithoutChildren.SubTreeNodeCount += 1
return preOrderTraversal
return preOrderTraversal, autoExpandedSpans
}
@@ -168,7 +199,13 @@ func GetSelectedSpans(uncollapsedSpans []string, selectedSpanID string, traceRoo
_, spansFromRootToNode := getPathFromRootToSelectedSpanId(rootNode, selectedSpanID, updatedUncollapsedSpans, isSelectedSpanIDUnCollapsed)
updatedUncollapsedSpans = append(updatedUncollapsedSpans, spansFromRootToNode...)
_preOrderTraversal := traverseTrace(rootNode, updatedUncollapsedSpans, 0, true, false, selectedSpanID)
_preOrderTraversal, _autoExpanded := traverseTrace(rootNode, updatedUncollapsedSpans, 0, true, false, selectedSpanID, -1, isSelectedSpanIDUnCollapsed, false)
// Merge auto-expanded spans into updatedUncollapsedSpans for returning in response
for _, spanID := range _autoExpanded {
if !slices.Contains(updatedUncollapsedSpans, spanID) {
updatedUncollapsedSpans = append(updatedUncollapsedSpans, spanID)
}
}
_selectedSpanIndex := findIndexForSelectedSpanFromPreOrder(_preOrderTraversal, selectedSpanID)
if _selectedSpanIndex != -1 {
@@ -212,3 +249,17 @@ func GetSelectedSpans(uncollapsedSpans []string, selectedSpanID string, traceRoo
return preOrderTraversal[startIndex:endIndex], updatedUncollapsedSpans, rootServiceName, rootServiceEntryPoint
}
func GetAllSpans(traceRoots []*model.Span) (spans []*model.Span, rootServiceName, rootEntryPoint string) {
for _, root := range traceRoots {
childSpans, _ := traverseTrace(root, nil, 0, true, false, "", -1, false, true)
spans = append(spans, childSpans...)
if rootServiceName == "" {
rootServiceName = root.ServiceName
}
if rootEntryPoint == "" {
rootEntryPoint = root.Name
}
}
return
}

View File

@@ -333,10 +333,14 @@ type GetWaterfallSpansForTraceWithMetadataParams struct {
SelectedSpanID string `json:"selectedSpanId"`
IsSelectedSpanIDUnCollapsed bool `json:"isSelectedSpanIDUnCollapsed"`
UncollapsedSpans []string `json:"uncollapsedSpans"`
Limit uint `json:"limit"`
}
type GetFlamegraphSpansForTraceParams struct {
SelectedSpanID string `json:"selectedSpanId"`
SelectedSpanID string `json:"selectedSpanId"`
Limit uint `json:"limit"`
BoundaryStartTS uint64 `json:"boundaryStartTsMilli"`
BoundaryEndTS uint64 `json:"boundarEndTsMilli"`
}
type SpanFilterParams struct {

View File

@@ -329,6 +329,7 @@ type GetWaterfallSpansForTraceWithMetadataResponse struct {
HasMissingSpans bool `json:"hasMissingSpans"`
// this is needed for frontend and query service sync
UncollapsedSpans []string `json:"uncollapsedSpans"`
HasMore bool `json:"hasMore"`
}
type GetFlamegraphSpansForTraceResponse struct {
@@ -336,6 +337,7 @@ type GetFlamegraphSpansForTraceResponse struct {
EndTimestampMillis uint64 `json:"endTimestampMillis"`
DurationNano uint64 `json:"durationNano"`
Spans [][]*FlamegraphSpan `json:"spans"`
HasMore bool `json:"hasMore"`
}
type OtelSpanRef struct {

View File

@@ -17,3 +17,7 @@ func Elapsed(funcName string, args map[string]interface{}) func() {
zap.L().Info("Elapsed time", zapFields...)
}
}
func MilliToNano(milliTS uint64) uint64 {
return milliTS * 1000_000
}

View File

@@ -1,19 +1,5 @@
import json
import os
import tempfile
import pytest
if not os.environ.get("DOCKER_CONFIG"):
os.environ["DOCKER_CONFIG"] = tempfile.mkdtemp(prefix="docker-config-")
os.environ.setdefault("DOCKER_CREDENTIAL_STORE", "")
os.environ.setdefault("DOCKER_CREDENTIAL_HELPER", "")
docker_config_path = os.path.join(os.environ["DOCKER_CONFIG"], "config.json")
if not os.path.exists(docker_config_path):
with open(docker_config_path, "w", encoding="utf-8") as config_file:
json.dump({"auths": {}, "credsStore": ""}, config_file)
pytest_plugins = [
"fixtures.auth",
"fixtures.clickhouse",
@@ -35,7 +21,6 @@ pytest_plugins = [
"fixtures.notification_channel",
"fixtures.alerts",
"fixtures.cloudintegrations",
"fixtures.jsontypeexporter",
]
@@ -73,7 +58,7 @@ def pytest_addoption(parser: pytest.Parser):
parser.addoption(
"--clickhouse-version",
action="store",
default="25.8.6",
default="25.5.6",
help="clickhouse version",
)
parser.addoption(
@@ -88,5 +73,3 @@ def pytest_addoption(parser: pytest.Parser):
default="v0.129.7",
help="schema migrator version",
)

View File

@@ -1,408 +0,0 @@
"""
Simpler version of jsontypeexporter for test fixtures.
This exports JSON type metadata to the path_types table by parsing JSON bodies
and extracting all paths with their types, similar to how the real jsontypeexporter works.
"""
import datetime
import json
from abc import ABC
from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, List, Optional, Set, Union
import numpy as np
import pytest
from fixtures import types
if TYPE_CHECKING:
from fixtures.logs import Logs
class JSONPathType(ABC):
"""Represents a JSON path with its type information"""
path: str
type: str
last_seen: np.uint64
def __init__(
self,
path: str,
type: str, # pylint: disable=redefined-builtin
last_seen: Optional[datetime.datetime] = None,
) -> None:
self.path = path
self.type = type
if last_seen is None:
last_seen = datetime.datetime.now()
self.last_seen = np.uint64(int(last_seen.timestamp() * 1e9))
def np_arr(self) -> np.array:
"""Return path type data as numpy array for database insertion"""
return np.array([self.path, self.type, self.last_seen])
# Constants matching jsontypeexporter
ARRAY_SEPARATOR = "[]." # Used in paths like "education[].name"
ARRAY_SUFFIX = "[]" # Used when traversing into array element objects
def _infer_array_type(elements: List[Any]) -> Optional[str]:
"""
Infer array type from array elements, matching jsontypeexporter's inferArrayMask logic.
Returns None if no valid type can be inferred.
"""
if len(elements) == 0:
return None
# Collect element types (matching Go: types := make([]pcommon.ValueType, 0, s.Len()))
types = []
for elem in elements:
if elem is None:
continue
if isinstance(elem, dict):
types.append("JSON")
elif isinstance(elem, str):
types.append("String")
elif isinstance(elem, bool):
types.append("Bool")
elif isinstance(elem, float):
types.append("Float64")
elif isinstance(elem, int):
types.append("Int64")
if len(types) == 0:
return None
# Get unique types (matching Go: unique := make(map[pcommon.ValueType]struct{}, len(types)))
unique = set(types)
# Classify types (matching Go logic)
has_json = "JSON" in unique
has_primitive = any(t in unique for t in ["String", "Bool", "Float64", "Int64"])
if has_json:
# If only JSON → Array(JSON) (no primitive types)
if not has_primitive:
return "Array(JSON)"
# If there's JSON + any primitive → Dynamic
return "Array(Dynamic)"
# ---- Primitive Type Resolution ----
if "String" in unique:
# Strings cannot coerce with any other primitive
if len(unique) > 1:
return "Array(Dynamic)"
return "Array(Nullable(String))"
if "Float64" in unique:
return "Array(Nullable(Float64))"
if "Int64" in unique:
return "Array(Nullable(Int64))"
if "Bool" in unique:
return "Array(Nullable(Bool))"
return "Array(Dynamic)"
def _python_type_to_clickhouse_type(value: Any) -> str:
"""
Convert Python type to ClickHouse JSON type string.
Maps Python types to ClickHouse JSON data types.
"""
if value is None:
return "String" # Default for null values
if isinstance(value, bool):
return "Bool"
elif isinstance(value, int):
return "Int64"
elif isinstance(value, float):
return "Float64"
elif isinstance(value, str):
return "String"
elif isinstance(value, list):
# Use the sophisticated array type inference
array_type = _infer_array_type(value)
return array_type if array_type else "Array(Dynamic)"
elif isinstance(value, dict):
return "JSON"
else:
return "String" # Default fallback
def _extract_json_paths(
obj: Any,
current_path: str = "",
path_types: Optional[Dict[str, Set[str]]] = None,
level: int = 0,
) -> Dict[str, Set[str]]:
"""
Recursively extract all paths and their types from a JSON object.
Matches jsontypeexporter's analyzePValue logic.
Args:
obj: The JSON object to traverse
current_path: Current path being built (e.g., "user.name")
path_types: Dictionary mapping paths to sets of types found
level: Current nesting level (for depth limiting)
Returns:
Dictionary mapping paths to sets of type strings
"""
if path_types is None:
path_types = {}
if obj is None:
if current_path:
if current_path not in path_types:
path_types[current_path] = set()
path_types[current_path].add("String") # Null defaults to String
return path_types
if isinstance(obj, dict):
# For objects, add the object itself and recurse into keys
if current_path:
if current_path not in path_types:
path_types[current_path] = set()
path_types[current_path].add("JSON")
for key, value in obj.items():
# Build the path for this key
if current_path:
new_path = f"{current_path}.{key}"
else:
new_path = key
# Recurse into the value
_extract_json_paths(value, new_path, path_types, level + 1)
elif isinstance(obj, list):
# Skip empty arrays
if len(obj) == 0:
return path_types
# Collect types from array elements (matching Go: types := make([]pcommon.ValueType, 0, s.Len()))
types = []
for item in obj:
if isinstance(item, dict):
# When traversing into array element objects, use ArraySuffix ([])
# This matches: prefix+ArraySuffix in the Go code
# Example: if current_path is "education", we use "education[]" to traverse into objects
array_prefix = current_path + ARRAY_SUFFIX if current_path else ""
for key, value in item.items():
if array_prefix:
# Use array separator: education[].name
array_path = f"{array_prefix}.{key}"
else:
array_path = key
# Recurse without increasing level (matching Go behavior)
_extract_json_paths(value, array_path, path_types, level)
types.append("JSON")
elif isinstance(item, list):
# Arrays inside arrays are not supported - skip the whole path
# Matching Go: e.logger.Error("arrays inside arrays are not supported!", ...); return nil
return path_types
elif isinstance(item, str):
types.append("String")
elif isinstance(item, bool):
types.append("Bool")
elif isinstance(item, float):
types.append("Float64")
elif isinstance(item, int):
types.append("Int64")
# Infer array type from collected types (matching Go: if mask := inferArrayMask(types); mask != 0)
if len(types) > 0:
array_type = _infer_array_type(obj)
if array_type and current_path:
if current_path not in path_types:
path_types[current_path] = set()
path_types[current_path].add(array_type)
else:
# Primitive value (string, number, bool)
if current_path:
if current_path not in path_types:
path_types[current_path] = set()
obj_type = _python_type_to_clickhouse_type(obj)
path_types[current_path].add(obj_type)
return path_types
def _parse_json_bodies_and_extract_paths(
json_bodies: List[str],
timestamp: Optional[datetime.datetime] = None,
) -> List[JSONPathType]:
"""
Parse JSON bodies and extract all paths with their types.
This mimics the behavior of jsontypeexporter.
Args:
json_bodies: List of JSON body strings to parse
timestamp: Timestamp to use for last_seen (defaults to now)
Returns:
List of JSONPathType objects with all discovered paths and types
"""
if timestamp is None:
timestamp = datetime.datetime.now()
# Aggregate all paths and their types across all JSON bodies
all_path_types: Dict[str, Set[str]] = {}
for json_body in json_bodies:
try:
parsed = json.loads(json_body)
_extract_json_paths(parsed, "", all_path_types, level=0)
except (json.JSONDecodeError, TypeError):
# Skip invalid JSON
continue
# Convert to list of JSONPathType objects
# Each path can have multiple types, so we create one JSONPathType per type
path_type_objects: List[JSONPathType] = []
for path, types_set in all_path_types.items():
for type_str in types_set:
path_type_objects.append(
JSONPathType(path=path, type=type_str, last_seen=timestamp)
)
return path_type_objects
@pytest.fixture(name="export_json_types", scope="function")
def export_json_types(
clickhouse: types.TestContainerClickhouse,
request: pytest.FixtureRequest, # To access migrator fixture
) -> Generator[Callable[[Union[List[JSONPathType], List[str], List[Any]]], None], Any, None]:
"""
Fixture for exporting JSON type metadata to the path_types table.
This is a simpler version of jsontypeexporter for test fixtures.
The function can accept:
1. List of JSONPathType objects (manual specification)
2. List of JSON body strings (auto-extract paths)
3. List of Logs objects (extract from body_json field)
Usage examples:
# Manual specification
export_json_types([
JSONPathType(path="user.name", type="String"),
JSONPathType(path="user.age", type="Int64"),
])
# Auto-extract from JSON strings
export_json_types([
'{"user": {"name": "alice", "age": 25}}',
'{"user": {"name": "bob", "age": 30}}',
])
# Auto-extract from Logs objects
export_json_types(logs_list)
"""
# Ensure migrator has run to create the table
try:
request.getfixturevalue("migrator")
except Exception:
# If migrator fixture is not available, that's okay - table might already exist
pass
def _export_json_types(
data: Union[List[JSONPathType], List[str], List[Any]], # List[Logs] but avoiding circular import
) -> None:
"""
Export JSON type metadata to signoz_metadata.distributed_json_path_types table.
This table stores path and type information for body JSON fields.
"""
path_types: List[JSONPathType] = []
if len(data) == 0:
return
# Determine input type and convert to JSONPathType list
first_item = data[0]
if isinstance(first_item, JSONPathType):
# Already JSONPathType objects
path_types = data # type: ignore
elif isinstance(first_item, str):
# List of JSON strings - parse and extract paths
path_types = _parse_json_bodies_and_extract_paths(data) # type: ignore
else:
# Assume it's a list of Logs objects - extract body_json
json_bodies: List[str] = []
for log in data: # type: ignore
# Try to get body_json attribute
if hasattr(log, "body_json") and log.body_json:
json_bodies.append(log.body_json)
elif hasattr(log, "body") and log.body:
# Fallback to body if body_json not available
try:
# Try to parse as JSON
json.loads(log.body)
json_bodies.append(log.body)
except (json.JSONDecodeError, TypeError):
pass
if json_bodies:
path_types = _parse_json_bodies_and_extract_paths(json_bodies)
if len(path_types) == 0:
return
clickhouse.conn.insert(
database="signoz_metadata",
table="distributed_json_path_types",
data=[path_type.np_arr() for path_type in path_types],
column_names=[
"path",
"type",
"last_seen",
],
)
yield _export_json_types
# Cleanup - truncate the local table after tests (following pattern from logs fixture)
clickhouse.conn.query(
f"TRUNCATE TABLE signoz_metadata.json_path_types ON CLUSTER '{clickhouse.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}' SYNC"
)
@pytest.fixture(name="export_promoted_paths", scope="function")
def export_promoted_paths(
clickhouse: types.TestContainerClickhouse,
request: pytest.FixtureRequest, # To access migrator fixture
) -> Generator[Callable[[List[str]], None], Any, None]:
"""
Fixture for exporting promoted JSON paths to the promoted paths table.
"""
# Ensure migrator has run to create the table
try:
request.getfixturevalue("migrator")
except Exception:
# If migrator fixture is not available, that's okay - table might already exist
pass
def _export_promoted_paths(paths: List[str]) -> None:
if len(paths) == 0:
return
now_ms = int(datetime.datetime.now().timestamp() * 1000)
rows = [(path, now_ms) for path in paths]
clickhouse.conn.insert(
database="signoz_metadata",
table="distributed_json_promoted_paths",
data=rows,
column_names=[
"path",
"created_at",
],
)
yield _export_promoted_paths
clickhouse.conn.query(
f"TRUNCATE TABLE signoz_metadata.json_promoted_paths ON CLUSTER '{clickhouse.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}' SYNC"
)

View File

@@ -100,8 +100,6 @@ class Logs(ABC):
severity_text: str
severity_number: np.uint8
body: str
body_json: str
body_json_promoted: str
attributes_string: dict[str, str]
attributes_number: dict[str, np.float64]
attributes_bool: dict[str, bool]
@@ -121,8 +119,6 @@ class Logs(ABC):
resources: dict[str, Any] = {},
attributes: dict[str, Any] = {},
body: str = "default body",
body_json: Optional[str] = None,
body_json_promoted: Optional[str] = None,
severity_text: str = "INFO",
trace_id: str = "",
span_id: str = "",
@@ -166,32 +162,6 @@ class Logs(ABC):
# Set body
self.body = body
# Set body_json - if body is JSON, parse and stringify it, otherwise use empty string
# ClickHouse accepts String input for JSON column
if body_json is not None:
self.body_json = body_json
else:
# Try to parse body as JSON, if successful use it, otherwise use empty string
try:
json.loads(body)
self.body_json = body
except (json.JSONDecodeError, TypeError):
self.body_json = ""
# Set body_json_promoted - must be valid JSON
# Tests will explicitly pass promoted column's content, but we validate it
if body_json_promoted is not None:
# Validate that it's valid JSON
try:
json.loads(body_json_promoted)
self.body_json_promoted = body_json_promoted
except (json.JSONDecodeError, TypeError):
# If invalid, default to empty JSON object
self.body_json_promoted = "{}"
else:
# Default to empty JSON object (valid JSON)
self.body_json_promoted = "{}"
# Process resources and attributes
self.resources_string = {k: str(v) for k, v in resources.items()}
@@ -349,8 +319,6 @@ class Logs(ABC):
self.severity_text,
self.severity_number,
self.body,
self.body_json,
self.body_json_promoted,
self.attributes_string,
self.attributes_number,
self.attributes_bool,
@@ -495,8 +463,6 @@ def insert_logs(
"severity_text",
"severity_number",
"body",
"body_json",
"body_json_promoted",
"attributes_string",
"attributes_number",
"attributes_bool",

View File

@@ -20,14 +20,12 @@ def migrator(
"""
def create() -> None:
# Hardcode version for new QB tests
version = "v0.129.13-rc.1"
version = request.config.getoption("--schema-migrator-version")
client = docker.from_env()
container = client.containers.run(
image=f"signoz/signoz-schema-migrator:{version}",
command=f"sync --replication=true --cluster-name=cluster --up= --dsn={clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN"]}",
environment={"ENABLE_LOGS_MIGRATIONS_V2": "1"},
detach=True,
auto_remove=False,
network=network.id,
@@ -46,7 +44,6 @@ def migrator(
container = client.containers.run(
image=f"signoz/signoz-schema-migrator:{version}",
command=f"async --replication=true --cluster-name=cluster --up= --dsn={clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN"]}",
environment={"ENABLE_LOGS_MIGRATIONS_V2": "1"},
detach=True,
auto_remove=False,
network=network.id,

View File

@@ -73,7 +73,6 @@ def signoz( # pylint: disable=too-many-arguments,too-many-positional-arguments
"SIGNOZ_ALERTMANAGER_SIGNOZ_POLL__INTERVAL": "5s",
"SIGNOZ_ALERTMANAGER_SIGNOZ_ROUTE_GROUP__WAIT": "1s",
"SIGNOZ_ALERTMANAGER_SIGNOZ_ROUTE_GROUP__INTERVAL": "5s",
"BODY_JSON_QUERY_ENABLED": "true",
}
| sqlstore.env
| clickhouse.env

File diff suppressed because it is too large Load Diff