feat(dashboards): add an axis to scatter plot thresholds

Assisted-by: Claude Opus 5.5

docs/api/openapi.yml

@@ -3702,6 +3702,13 @@ components:
           nullable: true
           type: number
       type: object
+    DashboardtypesAxisScale:
+      enum:
+      - auto
+      - linear
+      - log
+      - symlog
+      type: string
     DashboardtypesBarChartPanelSpec:
       properties:
         axes:
@@ -4444,6 +4451,7 @@ components:
           signoz/ListPanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesListPanelSpec'
           signoz/NumberPanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesNumberPanelSpec'
           signoz/PieChartPanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesPieChartPanelSpec'
+          signoz/ScatterPlotPanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesScatterPlotPanelSpec'
           signoz/TablePanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesTablePanelSpec'
           signoz/TextPanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesTextPanelSpec'
           signoz/TimeSeriesPanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesTimeSeriesPanelSpec'
@@ -4459,6 +4467,7 @@ components:
       - $ref: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesListPanelSpec'
       - $ref: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesTextPanelSpec'
       - $ref: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHeatmapPanelSpec'
+      - $ref: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesScatterPlotPanelSpec'
       type: object
     DashboardtypesPanelPluginKind:
       enum:
@@ -4472,6 +4481,7 @@ components:
       - signoz/ListPanel
       - signoz/TextPanel
       - signoz/HeatmapPanel
+      - signoz/ScatterPlotPanel
       type: string
     DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesAreaChartPanelSpec:
       properties:
@@ -4557,6 +4567,18 @@ components:
       - kind
       - spec
       type: object
+    DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesScatterPlotPanelSpec:
+      properties:
+        kind:
+          enum:
+          - signoz/ScatterPlotPanel
+          type: string
+        spec:
+          $ref: '#/components/schemas/DashboardtypesScatterPlotPanelSpec'
+      required:
+      - kind
+      - spec
+      type: object
     DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesTablePanelSpec:
       properties:
         kind:
@@ -4636,6 +4658,16 @@ components:
         visualization:
           $ref: '#/components/schemas/DashboardtypesBasicVisualization'
       type: object
+    DashboardtypesPointDiameter:
+      maximum: 40
+      minimum: 2
+      nullable: true
+      type: integer
+    DashboardtypesPointOpacity:
+      maximum: 1
+      minimum: 0.1
+      nullable: true
+      type: number
     DashboardtypesPostableDashboardV2:
       properties:
         generateName:
@@ -4824,6 +4856,104 @@ components:
       required:
       - queryValue
       type: object
+    DashboardtypesScatterPlotAxes:
+      properties:
+        x:
+          $ref: '#/components/schemas/DashboardtypesScatterPlotAxis'
+        "y":
+          $ref: '#/components/schemas/DashboardtypesScatterPlotAxis'
+      type: object
+    DashboardtypesScatterPlotAxis:
+      properties:
+        label:
+          description: Axis title. Empty draws none.
+          type: string
+        scale:
+          $ref: '#/components/schemas/DashboardtypesAxisScale'
+        softMax:
+          nullable: true
+          type: number
+        softMin:
+          nullable: true
+          type: number
+      type: object
+    DashboardtypesScatterPlotChartAppearance:
+      properties:
+        points:
+          $ref: '#/components/schemas/DashboardtypesScatterPlotPoints'
+      type: object
+    DashboardtypesScatterPlotDimensions:
+      properties:
+        colorBy:
+          description: Group-by label names (e.g. k8s.namespace.name) whose combined
+            values colour dots and drive the legend. Empty colours by every group-by
+            label.
+          items:
+            type: string
+          nullable: true
+          type: array
+        sizeBy:
+          description: Value column key that scales dot size. Empty draws every dot
+            at the default size.
+          type: string
+        x:
+          description: Value column key (queryName, or queryName.expression for a
+            multi-aggregation query) plotted on the x axis. Empty uses the first value
+            column.
+          type: string
+        "y":
+          description: Value column key plotted on the y axis. Empty uses the second
+            value column.
+          type: string
+      type: object
+    DashboardtypesScatterPlotPanelSpec:
+      properties:
+        axes:
+          $ref: '#/components/schemas/DashboardtypesScatterPlotAxes'
+        chartAppearance:
+          $ref: '#/components/schemas/DashboardtypesScatterPlotChartAppearance'
+        dimensions:
+          $ref: '#/components/schemas/DashboardtypesScatterPlotDimensions'
+        formatting:
+          $ref: '#/components/schemas/DashboardtypesTableFormatting'
+        legend:
+          $ref: '#/components/schemas/DashboardtypesLegend'
+        thresholds:
+          items:
+            $ref: '#/components/schemas/DashboardtypesScatterPlotThreshold'
+          nullable: true
+          type: array
+        visualization:
+          $ref: '#/components/schemas/DashboardtypesBasicVisualization'
+      type: object
+    DashboardtypesScatterPlotPoints:
+      properties:
+        maxSize:
+          $ref: '#/components/schemas/DashboardtypesPointDiameter'
+        minSize:
+          $ref: '#/components/schemas/DashboardtypesPointDiameter'
+        opacity:
+          $ref: '#/components/schemas/DashboardtypesPointOpacity'
+        size:
+          $ref: '#/components/schemas/DashboardtypesPointDiameter'
+      type: object
+    DashboardtypesScatterPlotThreshold:
+      properties:
+        axis:
+          $ref: '#/components/schemas/DashboardtypesThresholdAxis'
+        color:
+          type: string
+        label:
+          type: string
+        unit:
+          type: string
+        value:
+          format: double
+          type: number
+      required:
+      - value
+      - color
+      type: object
     DashboardtypesSource:
       enum:
       - user
@@ -4937,6 +5067,11 @@ components:
       - value
       - name
       type: object
+    DashboardtypesThresholdAxis:
+      enum:
+      - x
+      - "y"
+      type: string
     DashboardtypesThresholdFormat:
       enum:
       - text
@@ -16064,8 +16199,10 @@ paths:
                 $ref: '#/components/schemas/RenderErrorResponse'
           description: Internal Server Error
       security:
-      - api_key: []
-      - tokenizer: []
+      - api_key:
+        - VIEWER
+      - tokenizer:
+        - VIEWER
       summary: Get stats
       tags:
       - stats

frontend/src/pages/TraceDetailsV3/__tests__/getSpanDisplayData.test.ts

@@ -1,29 +0,0 @@
-import { SpanV3 } from 'types/api/trace/getTraceV3';
-
-import { getSpanDisplayData } from '../utils';
-
-describe('getSpanDisplayData', () => {
-	it('keeps only allow-listed fields', () => {
-		const span = {
-			span_id: 's1',
-			trace_id: 't1',
-			name: 'GET /api',
-			attributes: { 'http.method': 'GET' },
-			resource: { 'service.name': 'api' },
-			level: 1,
-			has_children: true,
-			sub_tree_node_count: 3,
-			'service.name': 'api',
-			time_unix: 1700000000,
-			formattedInput: '[]',
-		} as unknown as SpanV3;
-
-		expect(getSpanDisplayData(span)).toStrictEqual({
-			span_id: 's1',
-			trace_id: 't1',
-			name: 'GET /api',
-			attributes: { 'http.method': 'GET' },
-			resource: { 'service.name': 'api' },
-		});
-	});
-});

frontend/src/pages/TraceDetailsV3/utils.ts

@@ -1,5 +1,3 @@
-import type { SpantypesWaterfallSpanDTO } from 'api/generated/services/sigNoz.schemas';
-import { pick } from 'lodash-es';
 import { SpanV3 } from 'types/api/trace/getTraceV3';
 
 import { COLOR_BY_OPTIONS } from './constants';
@@ -48,43 +46,28 @@ export function hasInfraMetadata(span: SpanV3 | undefined): boolean {
 	return INFRA_METADATA_KEYS.some((key) => getSpanAttribute(span, key));
 }
 
-// Top-level fields shown in the Span Details DataViewer. Everything else on the
-// API response (e.g. waterfall rendering data) is hidden.
-const VISIBLE_SPAN_FIELDS_IN_DETAILS_VIEW = [
-	'span_id',
-	'trace_id',
-	'parent_span_id',
-	'name',
-	'kind_string',
-	'duration_nano',
-	'has_error',
-	'status_code',
-	'status_code_string',
-	'status_message',
-	'flags',
-	'is_remote',
-	'trace_state',
-	'attributes',
-	'resource',
-	'events',
-	'references',
-	'http_method',
-	'http_url',
-	'http_host',
-	'db_name',
-	'db_operation',
-	'external_http_method',
-	'external_http_url',
-	'response_status_code',
-	// Keys must exist on the waterfall DTO.
-] as const satisfies readonly (keyof SpantypesWaterfallSpanDTO)[];
+// Top-level fields that exist on the API response only to support waterfall
+// rendering. They have no value in the Span Details DataViewer. Drop the whole
+// constant + helper once the backend stops emitting them.
+const HIDDEN_SPAN_FIELDS_IN_DETAILS_VIEW: ReadonlySet<string> = new Set([
+	'sub_tree_node_count',
+	'has_children',
+	'level',
+	'service.name',
+]);
 
 /**
- * Shallow-copies the allow-listed span fields, for display in the Span Details
- * DataViewer.
+ * Shallow-copies the span with waterfall-only fields stripped, for display in
+ * the Span Details DataViewer.
  */
 export function getSpanDisplayData(span: SpanV3): Record<string, unknown> {
-	return pick(span, VISIBLE_SPAN_FIELDS_IN_DETAILS_VIEW);
+	const result: Record<string, unknown> = {};
+	for (const [key, value] of Object.entries(span)) {
+		if (!HIDDEN_SPAN_FIELDS_IN_DETAILS_VIEW.has(key)) {
+			result[key] = value;
+		}
+	}
+	return result;
 }
 
 /**

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/SigNoz/signoz-otel-collector v0.144.6
 	github.com/antlr4-go/antlr/v4 v4.13.1
 	github.com/antonmedv/expr v1.15.3
+	github.com/bytedance/sonic v1.14.1
 	github.com/cespare/xxhash/v2 v2.3.0
 	github.com/coreos/go-oidc/v3 v3.17.0
 	github.com/dgraph-io/ristretto/v2 v2.3.0
@@ -111,7 +112,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect
 	github.com/aws/smithy-go v1.24.2 // indirect
 	github.com/bytedance/gopkg v0.1.3 // indirect
-	github.com/bytedance/sonic v1.14.1 // indirect
 	github.com/bytedance/sonic/loader v0.3.0 // indirect
 	github.com/cloudwego/base64x v0.1.6 // indirect
 	github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect

pkg/apiserver/signozapiserver/statsreporter.go

@@ -4,12 +4,13 @@ import (
 	"net/http"
 
 	"github.com/SigNoz/signoz/pkg/http/handler"
+	"github.com/SigNoz/signoz/pkg/types"
 	"github.com/gorilla/mux"
 )
 
 func (provider *provider) addStatsReporterRoutes(router *mux.Router) error {
 	if err := router.Handle("/api/v1/stats", handler.New(
-		provider.authzMiddleware.OpenAccess(provider.statsHandler.Get),
+		provider.authzMiddleware.ViewAccess(provider.statsHandler.Get),
 		handler.OpenAPIDef{
 			ID:                  "GetStats",
 			Tags:                []string{"stats"},
@@ -22,7 +23,7 @@ func (provider *provider) addStatsReporterRoutes(router *mux.Router) error {
 			SuccessStatusCode:   http.StatusOK,
 			ErrorStatusCodes:    []int{},
 			Deprecated:          false,
-			SecuritySchemes:     newScopedSecuritySchemes(nil),
+			SecuritySchemes:     newSecuritySchemes(types.RoleViewer),
 		},
 	)).Methods(http.MethodGet).GetError(); err != nil {
 		return err

pkg/querier/consume.go

@@ -17,6 +17,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/errors"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/spantypes"
+	"github.com/SigNoz/signoz/pkg/types/telemetrystoretypes"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 )
 
@@ -39,18 +40,11 @@ func stripKeyAlias(name string) string {
 	return keyAliasRe.ReplaceAllString(name, "")
 }
 
-// unwrapVariant decodes a scan envelope: a chcol.Variant to its value, a chcol.JSON column to a map.
-// The attributes bag decodes flat so its dotted keys merge with the legacy attribute maps and a
-// scalar-and-object key stays two keys; every other JSON column decodes nested.
-func unwrapVariant(name string, val any) any {
-	switch v := val.(type) {
-	case chcol.Variant:
+// unwrapVariant returns the concrete value inside the chcol.Variant envelope the driver scans a
+// Dynamic column — a JSON path such as body_v2.level — into.
+func unwrapVariant(val any) any {
+	if v, ok := val.(chcol.Variant); ok {
 		return v.Any()
-	case chcol.JSON:
-		if name == "attributes" {
-			return spantypes.FlatJSONAttributes(v)
-		}
-		return v.NestedMap()
 	}
 	return val
 }
@@ -60,11 +54,11 @@ func unwrapVariant(name string, val any) any {
 // series. JSON goes through encoding/json for its sorted map keys: ClickHouse groups documents by
 // structure, so two rows it considers equal have to produce the same label.
 func labelValue(val any) string {
-	val = unwrapVariant("", val)
+	val = unwrapVariant(val)
 	if val == nil {
 		return ""
 	}
-	if v, ok := val.(map[string]any); ok {
+	if v, ok := val.(telemetrystoretypes.JSONValue); ok {
 		if raw, err := json.Marshal(v); err == nil {
 			return string(raw)
 		}
@@ -210,7 +204,7 @@ func readAsTimeSeries(rows driver.Rows, queryWindow *qbtypes.TimeRange, step qbt
 					Value: *val,
 				})
 
-			case *chcol.JSON, *chcol.Variant:
+			case *telemetrystoretypes.JSONValue, *chcol.Variant:
 				val := labelValue(derefValue(ptr))
 				lblVals = append(lblVals, val)
 				lblObjs = append(lblObjs, &qbtypes.Label{
@@ -484,7 +478,7 @@ func readAsScalar(rows driver.Rows, queryName string) (*qbtypes.ScalarData, erro
 		// 2. deref each slot into the output row
 		row := make([]any, len(scan))
 		for i, cell := range scan {
-			row[i] = unwrapVariant(cd[i].Name, derefValue(cell))
+			row[i] = unwrapVariant(derefValue(cell))
 		}
 		data = append(data, row)
 	}
@@ -542,7 +536,7 @@ func readAsRaw(rows driver.Rows, queryName string) (*qbtypes.RawData, error) {
 			name := stripKeyAlias(colNames[i])
 
 			// de-reference the typed pointer to any
-			val := unwrapVariant(name, reflect.ValueOf(cellPtr).Elem().Interface())
+			val := unwrapVariant(reflect.ValueOf(cellPtr).Elem().Interface())
 
 			// special-case: timestamp column
 			if name == "timestamp" || name == "timestamp_datetime" {
@@ -576,18 +570,17 @@ func readAsRaw(rows driver.Rows, queryName string) (*qbtypes.RawData, error) {
 // unified "attributes" and "resource" keys, and parses the stringified `events`
 // and `links` columns into structured slices. Raw DB columns are removed.
 //
-// The legacy attribute maps are merged over the JSON attributes bag, so maps win on collision.
+// The `attributes` JSON column is flattened in first and the legacy maps merged over it, so maps win on collision.
 func mergeSpanAttributeColumns(data map[string]any) {
 	attrStr, hasStr := data["attributes_string"]
 	attrNum, hasNum := data["attributes_number"]
 	attrBool, hasBool := data["attributes_bool"]
-	attributes, _ := data["attributes"].(map[string]any)
+	attrJSON, _ := data["attributes"].(telemetrystoretypes.JSONValue)
 	// todo(nitya): move to resource json
 	resStr, hasRes := data["resources_string"]
-	if hasStr || hasNum || hasBool || attributes != nil || hasRes {
-		if attributes == nil {
-			attributes = map[string]any{}
-		}
+	if hasStr || hasNum || hasBool || attrJSON != nil || hasRes {
+		attributes := make(map[string]any)
+		attrJSON.FlattenInto("", attributes)
 		if m, ok := attrStr.(map[string]string); ok {
 			for k, v := range m {
 				attributes[k] = v

pkg/querier/consume_test.go

@@ -3,9 +3,14 @@ package querier
 import (
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/ClickHouse/clickhouse-go/v2/lib/chcol"
+	cmock "github.com/SigNoz/clickhouse-go-mock"
+	"github.com/SigNoz/signoz/pkg/telemetrystore"
+	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/spantypes"
+	"github.com/SigNoz/signoz/pkg/types/telemetrystoretypes"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -78,52 +83,101 @@ func TestMergeSpanAttributeColumns_ParsesEventsAndLinks(t *testing.T) {
 	}
 }
 
-func TestUnwrapVariant(t *testing.T) {
-	j := chcol.NewJSON()
-	j.SetValueAtPath("level", "error")
-	j.SetValueAtPath("attrs.code", int64(500))
-
-	jNull := chcol.NewJSON()
-	jNull.SetValueAtPath("level", "error")
-	jNull.SetValueAtPath("absent", chcol.Variant{})
-
-	testCases := []struct {
-		name   string
-		column string
-		input  any
-		want   any
-	}{
-		{name: "VariantScalar", column: "", input: chcol.NewDynamicWithType("error", "String"), want: "error"},
-		{name: "EmptyDynamic", column: "", input: chcol.Dynamic{}, want: nil},
-		{name: "PlainValuePassthrough", column: "", input: uint64(3), want: uint64(3)},
-		{name: "JSONColumnNested", column: "body_v2", input: *j, want: map[string]any{"level": "error", "attrs": map[string]any{"code": int64(500)}}},
-		{name: "AttributesColumnFlat", column: "attributes", input: *j, want: map[string]any{"level": "error", "attrs.code": int64(500)}},
-		{name: "AttributesColumnDropsNullPaths", column: "attributes", input: *jNull, want: map[string]any{"level": "error"}},
+// A ClickHouse query can put a JSON column in the result of any request type — e.g.
+// `select * from signoz_logs.logs_v2` on a body_v2 stack, where `*` covers body_v2.
+func TestConsume_JSONColumn(t *testing.T) {
+	ts := time.Date(2026, 8, 14, 10, 0, 0, 0, time.UTC)
+	body := `{"level":"error","attrs":{"code":500}}`
+	wantBody := telemetrystoretypes.JSONValue{
+		"level": "error",
+		"attrs": map[string]any{"code": float64(500)},
 	}
 
-	for _, testCase := range testCases {
-		t.Run(testCase.name, func(t *testing.T) {
-			assert.Equal(t, testCase.want, unwrapVariant(testCase.column, testCase.input))
-		})
-	}
+	// the scalar reader reuses its scan slots across rows, so each row must still carry its own body
+	t.Run("scalar", func(t *testing.T) {
+		rows := telemetrystore.WrapRows(cmock.NewRows([]cmock.ColumnType{
+			{Name: "body_v2", Type: "JSON"},
+			{Name: "__result_0", Type: "UInt64"},
+		}, [][]any{{body, uint64(3)}, {`{"level":"warn"}`, uint64(1)}}))
+
+		payload, err := consume(rows, qbtypes.RequestTypeScalar, nil, qbtypes.Step{}, "A")
+		require.NoError(t, err)
+
+		data := payload.(*qbtypes.ScalarData)
+		require.Len(t, data.Data, 2)
+		assert.Equal(t, wantBody, data.Data[0][0])
+		assert.Equal(t, uint64(3), data.Data[0][1])
+		assert.Equal(t, telemetrystoretypes.JSONValue{"level": "warn"}, data.Data[1][0])
+		assert.Equal(t, uint64(1), data.Data[1][1])
+	})
+
+	t.Run("time series", func(t *testing.T) {
+		rows := telemetrystore.WrapRows(cmock.NewRows([]cmock.ColumnType{
+			{Name: "ts", Type: "DateTime"},
+			{Name: "body_v2", Type: "JSON"},
+			{Name: "__result_0", Type: "UInt64"},
+		}, [][]any{{ts, body, uint64(3)}}))
+
+		payload, err := consume(rows, qbtypes.RequestTypeTimeSeries, nil, qbtypes.Step{}, "A")
+		require.NoError(t, err)
+
+		data := payload.(*qbtypes.TimeSeriesData)
+		require.Len(t, data.Aggregations, 1)
+		require.Len(t, data.Aggregations[0].Series, 1)
+		require.Len(t, data.Aggregations[0].Series[0].Values, 1)
+		assert.Equal(t, float64(3), data.Aggregations[0].Series[0].Values[0].Value)
+	})
+
+	// grouping by a JSON column is legal in ClickHouse, so each document has to label its own
+	// series rather than being dropped, which would merge every group into one
+	t.Run("time series grouped by the JSON column", func(t *testing.T) {
+		rows := telemetrystore.WrapRows(cmock.NewRows([]cmock.ColumnType{
+			{Name: "ts", Type: "DateTime"},
+			{Name: "body_v2", Type: "JSON"},
+			{Name: "__result_0", Type: "UInt64"},
+		}, [][]any{
+			{ts, `{"level":"error"}`, uint64(7)},
+			{ts, `{"level":"warn"}`, uint64(2)},
+		}))
+
+		payload, err := consume(rows, qbtypes.RequestTypeTimeSeries, nil, qbtypes.Step{}, "A")
+		require.NoError(t, err)
+
+		data := payload.(*qbtypes.TimeSeriesData)
+		require.Len(t, data.Aggregations, 1)
+		require.Len(t, data.Aggregations[0].Series, 2)
+
+		got := map[string]float64{}
+		for _, series := range data.Aggregations[0].Series {
+			require.Len(t, series.Labels, 1)
+			require.Len(t, series.Values, 1)
+			got[series.Labels[0].Value.(string)] = series.Values[0].Value
+		}
+		assert.Equal(t, map[string]float64{`{"level":"error"}`: 7, `{"level":"warn"}`: 2}, got)
+	})
+
+	t.Run("raw", func(t *testing.T) {
+		rows := telemetrystore.WrapRows(cmock.NewRows([]cmock.ColumnType{
+			{Name: "timestamp", Type: "DateTime"},
+			{Name: "body_v2", Type: "JSON"},
+		}, [][]any{{ts, body}}))
+
+		payload, err := consume(rows, qbtypes.RequestTypeRaw, nil, qbtypes.Step{}, "A")
+		require.NoError(t, err)
+
+		data := payload.(*qbtypes.RawData)
+		require.Len(t, data.Rows, 1)
+		assert.Equal(t, ts, data.Rows[0].Timestamp.UTC())
+		assert.Equal(t, wantBody, data.Rows[0].Data["body_v2"])
+	})
 }
 
-func TestLabelValue(t *testing.T) {
-	testCases := []struct {
-		name  string
-		input any
-		want  string
-	}{
-		{name: "Nil", input: nil, want: ""},
-		{name: "Variant", input: chcol.NewDynamicWithType("error", "String"), want: "error"},
-		{name: "JSONMap", input: map[string]any{"level": "error", "attrs": map[string]any{"code": 500}}, want: `{"attrs":{"code":500},"level":"error"}`},
-	}
-
-	for _, testCase := range testCases {
-		t.Run(testCase.name, func(t *testing.T) {
-			assert.Equal(t, testCase.want, labelValue(testCase.input))
-		})
-	}
+// A JSON path (e.g. `body_v2.level`) comes back as a Dynamic column, which the driver scans
+// into a chcol.Variant envelope rather than the value itself.
+func TestUnwrapVariant(t *testing.T) {
+	assert.Equal(t, "error", unwrapVariant(chcol.NewDynamicWithType("error", "String")))
+	assert.Nil(t, unwrapVariant(chcol.Dynamic{}))
+	assert.Equal(t, uint64(3), unwrapVariant(uint64(3)))
 }
 
 func TestMergeSpanAttributeColumns_EmptyEventsAndLinks(t *testing.T) {
@@ -151,9 +205,12 @@ func TestMergeSpanAttributeColumns_JSONColumn(t *testing.T) {
 		want map[string]any
 	}{
 		{
-			name: "JSONOnly_FlatPathsPreserveTypes",
+			name: "JSONOnly_FlattensNestedPaths_PreservesTypes",
 			data: map[string]any{
-				"attributes": map[string]any{"http.route": "/api/pay", "http.retry.count": float64(3), "cache.hit": true},
+				"attributes": telemetrystoretypes.JSONValue{
+					"http":      map[string]any{"route": "/api/pay", "retry": map[string]any{"count": float64(3)}},
+					"cache.hit": true,
+				},
 			},
 			want: map[string]any{"http.route": "/api/pay", "http.retry.count": float64(3), "cache.hit": true},
 		},
@@ -162,24 +219,17 @@ func TestMergeSpanAttributeColumns_JSONColumn(t *testing.T) {
 			data: map[string]any{
 				"attributes_string": map[string]string{"http.route": "/old", "only.map": "m"},
 				"attributes_number": map[string]float64{"http.status": 500},
-				"attributes":        map[string]any{"http.route": "/new", "only.json": "j"},
+				"attributes":        telemetrystoretypes.JSONValue{"http": map[string]any{"route": "/new"}, "only.json": "j"},
 			},
 			want: map[string]any{"http.route": "/old", "only.map": "m", "http.status": float64(500), "only.json": "j"},
 		},
-		{
-			name: "ScalarAndObjectKey_BothSurvive",
-			data: map[string]any{
-				"attributes": map[string]any{"db.function": "refresh", "db.function.arg": float64(2)},
-			},
-			want: map[string]any{"db.function": "refresh", "db.function.arg": float64(2)},
-		},
 		{
 			name: "MapOnly_EmptyJSONDoc_KeepsMapValues",
 			data: map[string]any{
 				"attributes_string": map[string]string{"http.route": "/map"},
 				"attributes_number": map[string]float64{"http.status": 200},
 				"attributes_bool":   map[string]bool{"cache.hit": true},
-				"attributes":        map[string]any{},
+				"attributes":        telemetrystoretypes.JSONValue{},
 			},
 			want: map[string]any{"http.route": "/map", "http.status": float64(200), "cache.hit": true},
 		},
@@ -187,53 +237,60 @@ func TestMergeSpanAttributeColumns_JSONColumn(t *testing.T) {
 			name: "MapOnly_NilJSON_BehavesAsAbsent",
 			data: map[string]any{
 				"attributes_string": map[string]string{"http.route": "/map"},
-				"attributes":        map[string]any(nil),
+				"attributes":        telemetrystoretypes.JSONValue(nil),
 			},
 			want: map[string]any{"http.route": "/map"},
 		},
 		{
 			name: "Arrays_StayLeafValues",
 			data: map[string]any{
-				"attributes": map[string]any{"http.tags": []any{"a", "b"}, "http.codes": []any{float64(1), float64(2)}},
+				"attributes": telemetrystoretypes.JSONValue{"http": map[string]any{"tags": []any{"a", "b"}, "codes": []any{float64(1), float64(2)}}},
 			},
 			want: map[string]any{"http.tags": []any{"a", "b"}, "http.codes": []any{float64(1), float64(2)}},
 		},
 		{
 			name: "TopLevelArrayOfMaps_StaysNativeLeaf",
 			data: map[string]any{
-				"attributes": map[string]any{"key": []any{map[string]any{"a": float64(1)}, map[string]any{"b": float64(2)}}},
+				"attributes": telemetrystoretypes.JSONValue{"key": []any{map[string]any{"a": float64(1)}, map[string]any{"b": float64(2)}}},
 			},
 			want: map[string]any{"key": []any{map[string]any{"a": float64(1)}, map[string]any{"b": float64(2)}}},
 		},
 		{
-			name: "ArrayOfMapsAtDottedPath_StaysNativeLeaf",
+			name: "NestedArrayOfMaps_StaysNativeLeaf_NoIndexPaths",
 			data: map[string]any{
-				"attributes": map[string]any{"http.items": []any{map[string]any{"a": float64(1)}}},
+				"attributes": telemetrystoretypes.JSONValue{"http": map[string]any{"items": []any{map[string]any{"a": float64(1)}}}},
 			},
 			want: map[string]any{"http.items": []any{map[string]any{"a": float64(1)}}},
 		},
 		{
-			name: "DualWritten_IndexKeysAndJSONArrayCoexist",
+			name: "DualWritten_NestedArray_IndexKeysAndJSONArrayCoexist",
 			data: map[string]any{
 				"attributes_number": map[string]float64{"http.items.0.a": 1},
-				"attributes":        map[string]any{"http.items": []any{map[string]any{"a": float64(1)}}},
+				"attributes":        telemetrystoretypes.JSONValue{"http": map[string]any{"items": []any{map[string]any{"a": float64(1)}}}},
 			},
 			want: map[string]any{"http.items.0.a": float64(1), "http.items": []any{map[string]any{"a": float64(1)}}},
 		},
 		{
 			name: "JSONNull_KeptAsNil",
 			data: map[string]any{
-				"attributes": map[string]any{"k": nil},
+				"attributes": telemetrystoretypes.JSONValue{"k": nil},
 			},
 			want: map[string]any{"k": nil},
 		},
 		{
-			name: "ScalarValueStaysScalar",
+			name: "KeyIsLeafValue_NotFlattened",
 			data: map[string]any{
-				"attributes": map[string]any{"http": "plaintext"},
+				"attributes": telemetrystoretypes.JSONValue{"http": "plaintext"},
 			},
 			want: map[string]any{"http": "plaintext"},
 		},
+		{
+			name: "KeyIsParent_FlattensToDottedPath",
+			data: map[string]any{
+				"attributes": telemetrystoretypes.JSONValue{"http": map[string]any{"route": "/a"}},
+			},
+			want: map[string]any{"http.route": "/a"},
+		},
 	}
 
 	for _, testCase := range testCases {

pkg/querier/postprocess.go

@@ -16,6 +16,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/querybuilder"
 	"github.com/SigNoz/signoz/pkg/types/featuretypes"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/SigNoz/signoz/pkg/types/telemetrystoretypes"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 	"github.com/SigNoz/signoz/pkg/valuer"
 )
@@ -1201,7 +1202,7 @@ func (q *querier) postProcessLogBody(ctx context.Context, orgID valuer.UUID, res
 // carried one. Anything that is not a decoded document — the legacy string body, a NULL cell —
 // is legal under these names and left alone.
 func stripEmptyBodyMessage(val any) {
-	bodyMap, ok := val.(map[string]any)
+	bodyMap, ok := val.(telemetrystoretypes.JSONValue)
 	if !ok {
 		return
 	}

pkg/query-service/app/clickhouseReader/reader.go

@@ -62,6 +62,8 @@ const (
 	primaryNamespace          = "clickhouse"
 	archiveNamespace          = "clickhouse-archive"
 	signozTraceDBName         = "signoz_traces"
+	signozHistoryDBName       = "signoz_analytics"
+	ruleStateHistoryTableName = "distributed_rule_state_history_v0"
 	signozDurationMVTable     = "distributed_durationSort"
 	signozUsageExplorerTable  = "distributed_usage_explorer"
 	signozSpansTable          = "distributed_signoz_spans"
@@ -4482,6 +4484,483 @@ func (r *ClickHouseReader) GetSpanAttributeKeysByNames(ctx context.Context, name
 	return response, nil
 }
 
+func (r *ClickHouseReader) AddRuleStateHistory(ctx context.Context, ruleStateHistory []model.RuleStateHistory) error {
+	ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
+		instrumentationtypes.CodeNamespace:    "clickhouse-reader",
+		instrumentationtypes.CodeFunctionName: "AddRuleStateHistory",
+	})
+	var statement driver.Batch
+	var err error
+
+	defer func() {
+		if statement != nil {
+			statement.Abort()
+		}
+	}()
+
+	statement, err = r.db.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.%s (rule_id, rule_name, overall_state, overall_state_changed, state, state_changed, unix_milli, labels, fingerprint, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
+		signozHistoryDBName, ruleStateHistoryTableName))
+
+	if err != nil {
+		return err
+	}
+
+	for _, history := range ruleStateHistory {
+		err = statement.Append(history.RuleID, history.RuleName, history.OverallState, history.OverallStateChanged, history.State, history.StateChanged, history.UnixMilli, history.Labels, history.Fingerprint, history.Value)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = statement.Send()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (r *ClickHouseReader) GetLastSavedRuleStateHistory(ctx context.Context, ruleID string) ([]model.RuleStateHistory, error) {
+	ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
+		instrumentationtypes.CodeNamespace:    "clickhouse-reader",
+		instrumentationtypes.CodeFunctionName: "GetLastSavedRuleStateHistory",
+	})
+	query := fmt.Sprintf("SELECT * FROM %s.%s WHERE rule_id = '%s' AND state_changed = true ORDER BY unix_milli DESC LIMIT 1 BY fingerprint",
+		signozHistoryDBName, ruleStateHistoryTableName, ruleID)
+
+	history := []model.RuleStateHistory{}
+	err := r.db.Select(ctx, &history, query)
+	if err != nil {
+		return nil, err
+	}
+	return history, nil
+}
+
+func (r *ClickHouseReader) ReadRuleStateHistoryByRuleID(
+	ctx context.Context, ruleID string, params *model.QueryRuleStateHistory) (*model.RuleStateTimeline, error) {
+
+	ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
+		instrumentationtypes.CodeNamespace:    "clickhouse-reader",
+		instrumentationtypes.CodeFunctionName: "ReadRuleStateHistoryByRuleID",
+	})
+	var conditions []string
+
+	conditions = append(conditions, fmt.Sprintf("rule_id = '%s'", ruleID))
+
+	conditions = append(conditions, fmt.Sprintf("unix_milli >= %d AND unix_milli < %d", params.Start, params.End))
+
+	if params.State != "" {
+		conditions = append(conditions, fmt.Sprintf("state = '%s'", params.State))
+	}
+
+	if params.Filters != nil && len(params.Filters.Items) != 0 {
+		for _, item := range params.Filters.Items {
+			toFormat := item.Value
+			op := v3.FilterOperator(strings.ToLower(strings.TrimSpace(string(item.Operator))))
+			if op == v3.FilterOperatorContains || op == v3.FilterOperatorNotContains {
+				toFormat = fmt.Sprintf("%%%s%%", toFormat)
+			}
+			fmtVal := utils.ClickHouseFormattedValue(toFormat)
+			switch op {
+			case v3.FilterOperatorEqual:
+				conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') = %s", item.Key.Key, fmtVal))
+			case v3.FilterOperatorNotEqual:
+				conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') != %s", item.Key.Key, fmtVal))
+			case v3.FilterOperatorIn:
+				conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') IN %s", item.Key.Key, fmtVal))
+			case v3.FilterOperatorNotIn:
+				conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') NOT IN %s", item.Key.Key, fmtVal))
+			case v3.FilterOperatorLike:
+				conditions = append(conditions, fmt.Sprintf("like(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal))
+			case v3.FilterOperatorNotLike:
+				conditions = append(conditions, fmt.Sprintf("notLike(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal))
+			case v3.FilterOperatorRegex:
+				conditions = append(conditions, fmt.Sprintf("match(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal))
+			case v3.FilterOperatorNotRegex:
+				conditions = append(conditions, fmt.Sprintf("not match(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal))
+			case v3.FilterOperatorGreaterThan:
+				conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') > %s", item.Key.Key, fmtVal))
+			case v3.FilterOperatorGreaterThanOrEq:
+				conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') >= %s", item.Key.Key, fmtVal))
+			case v3.FilterOperatorLessThan:
+				conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') < %s", item.Key.Key, fmtVal))
+			case v3.FilterOperatorLessThanOrEq:
+				conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') <= %s", item.Key.Key, fmtVal))
+			case v3.FilterOperatorContains:
+				conditions = append(conditions, fmt.Sprintf("like(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal))
+			case v3.FilterOperatorNotContains:
+				conditions = append(conditions, fmt.Sprintf("notLike(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal))
+			case v3.FilterOperatorExists:
+				conditions = append(conditions, fmt.Sprintf("has(JSONExtractKeys(labels), '%s')", item.Key.Key))
+			case v3.FilterOperatorNotExists:
+				conditions = append(conditions, fmt.Sprintf("not has(JSONExtractKeys(labels), '%s')", item.Key.Key))
+			default:
+				return nil, fmt.Errorf("unsupported filter operator")
+			}
+		}
+	}
+	whereClause := strings.Join(conditions, " AND ")
+
+	query := fmt.Sprintf("SELECT * FROM %s.%s WHERE %s ORDER BY unix_milli %s LIMIT %d OFFSET %d",
+		signozHistoryDBName, ruleStateHistoryTableName, whereClause, params.Order, params.Limit, params.Offset)
+
+	history := []model.RuleStateHistory{}
+	r.logger.Debug("rule state history query", "query", query)
+	err := r.db.Select(ctx, &history, query)
+	if err != nil {
+		r.logger.Error("Error while reading rule state history", errorsV2.Attr(err))
+		return nil, err
+	}
+
+	var total uint64
+	r.logger.Debug("rule state history total query", "query", fmt.Sprintf("SELECT count(*) FROM %s.%s WHERE %s",
+		signozHistoryDBName, ruleStateHistoryTableName, whereClause))
+	err = r.db.QueryRow(ctx, fmt.Sprintf("SELECT count(*) FROM %s.%s WHERE %s",
+		signozHistoryDBName, ruleStateHistoryTableName, whereClause)).Scan(&total)
+	if err != nil {
+		return nil, err
+	}
+
+	labelsQuery := fmt.Sprintf("SELECT DISTINCT labels FROM %s.%s WHERE rule_id = $1",
+		signozHistoryDBName, ruleStateHistoryTableName)
+	rows, err := r.db.Query(ctx, labelsQuery, ruleID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	labelsMap := make(map[string][]string)
+	for rows.Next() {
+		var rawLabel string
+		err = rows.Scan(&rawLabel)
+		if err != nil {
+			return nil, err
+		}
+		label := map[string]string{}
+		err = json.Unmarshal([]byte(rawLabel), &label)
+		if err != nil {
+			return nil, err
+		}
+		for k, v := range label {
+			labelsMap[k] = append(labelsMap[k], v)
+		}
+	}
+
+	timeline := &model.RuleStateTimeline{
+		Items:  history,
+		Total:  total,
+		Labels: labelsMap,
+	}
+
+	return timeline, nil
+}
+
+func (r *ClickHouseReader) ReadRuleStateHistoryTopContributorsByRuleID(
+	ctx context.Context, ruleID string, params *model.QueryRuleStateHistory) ([]model.RuleStateHistoryContributor, error) {
+	ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
+		instrumentationtypes.CodeNamespace:    "clickhouse-reader",
+		instrumentationtypes.CodeFunctionName: "ReadRuleStateHistoryTopContributorsByRuleID",
+	})
+	query := fmt.Sprintf(`SELECT
+		fingerprint,
+		any(labels) as labels,
+		count(*) as count
+	FROM %s.%s
+	WHERE rule_id = '%s' AND (state_changed = true) AND (state = '%s') AND unix_milli >= %d AND unix_milli <= %d
+	GROUP BY fingerprint
+	HAVING labels != '{}'
+	ORDER BY count DESC`,
+		signozHistoryDBName, ruleStateHistoryTableName, ruleID, model.StateFiring.String(), params.Start, params.End)
+
+	r.logger.Debug("rule state history top contributors query", "query", query)
+	contributors := []model.RuleStateHistoryContributor{}
+	err := r.db.Select(ctx, &contributors, query)
+	if err != nil {
+		r.logger.Error("Error while reading rule state history", errorsV2.Attr(err))
+		return nil, err
+	}
+
+	return contributors, nil
+}
+
+func (r *ClickHouseReader) GetOverallStateTransitions(ctx context.Context, ruleID string, params *model.QueryRuleStateHistory) ([]model.ReleStateItem, error) {
+
+	ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
+		instrumentationtypes.CodeNamespace:    "clickhouse-reader",
+		instrumentationtypes.CodeFunctionName: "GetOverallStateTransitions",
+	})
+	tmpl := `WITH firing_events AS (
+    SELECT
+        rule_id,
+        state,
+        unix_milli AS firing_time
+    FROM %s.%s
+    WHERE overall_state = '` + model.StateFiring.String() + `'
+      AND overall_state_changed = true
+      AND rule_id IN ('%s')
+	  AND unix_milli >= %d AND unix_milli <= %d
+),
+resolution_events AS (
+    SELECT
+        rule_id,
+        state,
+        unix_milli AS resolution_time
+    FROM %s.%s
+    WHERE overall_state = '` + model.StateInactive.String() + `'
+      AND overall_state_changed = true
+      AND rule_id IN ('%s')
+	  AND unix_milli >= %d AND unix_milli <= %d
+),
+matched_events AS (
+    SELECT
+        f.rule_id,
+        f.state,
+        f.firing_time,
+        MIN(r.resolution_time) AS resolution_time
+    FROM firing_events f
+    LEFT JOIN resolution_events r
+        ON f.rule_id = r.rule_id
+    WHERE r.resolution_time > f.firing_time
+    GROUP BY f.rule_id, f.state, f.firing_time
+)
+SELECT *
+FROM matched_events
+ORDER BY firing_time ASC;`
+
+	query := fmt.Sprintf(tmpl,
+		signozHistoryDBName, ruleStateHistoryTableName, ruleID, params.Start, params.End,
+		signozHistoryDBName, ruleStateHistoryTableName, ruleID, params.Start, params.End)
+
+	r.logger.Debug("overall state transitions query", "query", query)
+
+	transitions := []model.RuleStateTransition{}
+	err := r.db.Select(ctx, &transitions, query)
+	if err != nil {
+		return nil, err
+	}
+
+	stateItems := []model.ReleStateItem{}
+
+	for idx, item := range transitions {
+		start := item.FiringTime
+		end := item.ResolutionTime
+		stateItems = append(stateItems, model.ReleStateItem{
+			State: item.State,
+			Start: start,
+			End:   end,
+		})
+		if idx < len(transitions)-1 {
+			nextStart := transitions[idx+1].FiringTime
+			if nextStart > end {
+				stateItems = append(stateItems, model.ReleStateItem{
+					State: model.StateInactive,
+					Start: end,
+					End:   nextStart,
+				})
+			}
+		}
+	}
+
+	// fetch the most recent overall_state from the table
+	var state model.AlertState
+	stateQuery := fmt.Sprintf("SELECT state FROM %s.%s WHERE rule_id = '%s' AND unix_milli <= %d ORDER BY unix_milli DESC LIMIT 1",
+		signozHistoryDBName, ruleStateHistoryTableName, ruleID, params.End)
+	if err := r.db.QueryRow(ctx, stateQuery).Scan(&state); err != nil {
+		if err != sql.ErrNoRows {
+			return nil, err
+		}
+		state = model.StateInactive
+	}
+
+	if len(transitions) == 0 {
+		// no transitions found, it is either firing or inactive for whole time range
+		stateItems = append(stateItems, model.ReleStateItem{
+			State: state,
+			Start: params.Start,
+			End:   params.End,
+		})
+	} else {
+		// there were some transitions, we need to add the last state at the end
+		if state == model.StateInactive {
+			stateItems = append(stateItems, model.ReleStateItem{
+				State: model.StateInactive,
+				Start: transitions[len(transitions)-1].ResolutionTime,
+				End:   params.End,
+			})
+		} else {
+			// fetch the most recent firing event from the table in the given time range
+			var firingTime int64
+			firingQuery := fmt.Sprintf(`
+			SELECT
+				unix_milli
+			FROM %s.%s
+			WHERE rule_id = '%s' AND overall_state_changed = true AND overall_state = '%s' AND unix_milli <= %d
+			ORDER BY unix_milli DESC LIMIT 1`, signozHistoryDBName, ruleStateHistoryTableName, ruleID, model.StateFiring.String(), params.End)
+			if err := r.db.QueryRow(ctx, firingQuery).Scan(&firingTime); err != nil {
+				return nil, err
+			}
+			stateItems = append(stateItems, model.ReleStateItem{
+				State: model.StateInactive,
+				Start: transitions[len(transitions)-1].ResolutionTime,
+				End:   firingTime,
+			})
+			stateItems = append(stateItems, model.ReleStateItem{
+				State: model.StateFiring,
+				Start: firingTime,
+				End:   params.End,
+			})
+		}
+	}
+	return stateItems, nil
+}
+
+func (r *ClickHouseReader) GetAvgResolutionTime(ctx context.Context, ruleID string, params *model.QueryRuleStateHistory) (float64, error) {
+
+	ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
+		instrumentationtypes.CodeNamespace:    "clickhouse-reader",
+		instrumentationtypes.CodeFunctionName: "GetAvgResolutionTime",
+	})
+	tmpl := `
+WITH firing_events AS (
+    SELECT
+        rule_id,
+        state,
+        unix_milli AS firing_time
+    FROM %s.%s
+    WHERE overall_state = '` + model.StateFiring.String() + `'
+      AND overall_state_changed = true
+      AND rule_id IN ('%s')
+	  AND unix_milli >= %d AND unix_milli <= %d
+),
+resolution_events AS (
+    SELECT
+        rule_id,
+        state,
+        unix_milli AS resolution_time
+    FROM %s.%s
+    WHERE overall_state = '` + model.StateInactive.String() + `'
+      AND overall_state_changed = true
+      AND rule_id IN ('%s')
+	  AND unix_milli >= %d AND unix_milli <= %d
+),
+matched_events AS (
+    SELECT
+        f.rule_id,
+        f.state,
+        f.firing_time,
+        MIN(r.resolution_time) AS resolution_time
+    FROM firing_events f
+    LEFT JOIN resolution_events r
+        ON f.rule_id = r.rule_id
+    WHERE r.resolution_time > f.firing_time
+    GROUP BY f.rule_id, f.state, f.firing_time
+)
+SELECT AVG(resolution_time - firing_time) / 1000 AS avg_resolution_time
+FROM matched_events;
+`
+
+	query := fmt.Sprintf(tmpl,
+		signozHistoryDBName, ruleStateHistoryTableName, ruleID, params.Start, params.End,
+		signozHistoryDBName, ruleStateHistoryTableName, ruleID, params.Start, params.End)
+
+	r.logger.Debug("avg resolution time query", "query", query)
+	var avgResolutionTime float64
+	err := r.db.QueryRow(ctx, query).Scan(&avgResolutionTime)
+	if err != nil {
+		return 0, err
+	}
+
+	return avgResolutionTime, nil
+}
+
+func (r *ClickHouseReader) GetAvgResolutionTimeByInterval(ctx context.Context, ruleID string, params *model.QueryRuleStateHistory) (*v3.Series, error) {
+
+	step := common.MinAllowedStepInterval(params.Start, params.End)
+
+	tmpl := `
+WITH firing_events AS (
+    SELECT
+        rule_id,
+        state,
+        unix_milli AS firing_time
+    FROM %s.%s
+    WHERE overall_state = '` + model.StateFiring.String() + `'
+      AND overall_state_changed = true
+      AND rule_id IN ('%s')
+	  AND unix_milli >= %d AND unix_milli <= %d
+),
+resolution_events AS (
+    SELECT
+        rule_id,
+        state,
+        unix_milli AS resolution_time
+    FROM %s.%s
+    WHERE overall_state = '` + model.StateInactive.String() + `'
+      AND overall_state_changed = true
+      AND rule_id IN ('%s')
+	  AND unix_milli >= %d AND unix_milli <= %d
+),
+matched_events AS (
+    SELECT
+        f.rule_id,
+        f.state,
+        f.firing_time,
+        MIN(r.resolution_time) AS resolution_time
+    FROM firing_events f
+    LEFT JOIN resolution_events r
+        ON f.rule_id = r.rule_id
+    WHERE r.resolution_time > f.firing_time
+    GROUP BY f.rule_id, f.state, f.firing_time
+)
+SELECT toStartOfInterval(toDateTime(firing_time / 1000), INTERVAL %d SECOND) AS ts, AVG(resolution_time - firing_time) / 1000 AS avg_resolution_time
+FROM matched_events
+GROUP BY ts
+ORDER BY ts ASC;`
+
+	query := fmt.Sprintf(tmpl,
+		signozHistoryDBName, ruleStateHistoryTableName, ruleID, params.Start, params.End,
+		signozHistoryDBName, ruleStateHistoryTableName, ruleID, params.Start, params.End, step)
+
+	r.logger.Debug("avg resolution time by interval query", "query", query)
+	result, err := r.GetTimeSeriesResultV3(ctx, query)
+	if err != nil || len(result) == 0 {
+		return nil, err
+	}
+
+	return result[0], nil
+}
+
+func (r *ClickHouseReader) GetTotalTriggers(ctx context.Context, ruleID string, params *model.QueryRuleStateHistory) (uint64, error) {
+	ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
+		instrumentationtypes.CodeNamespace:    "clickhouse-reader",
+		instrumentationtypes.CodeFunctionName: "GetTotalTriggers",
+	})
+	query := fmt.Sprintf("SELECT count(*) FROM %s.%s WHERE rule_id = '%s' AND (state_changed = true) AND (state = '%s') AND unix_milli >= %d AND unix_milli <= %d",
+		signozHistoryDBName, ruleStateHistoryTableName, ruleID, model.StateFiring.String(), params.Start, params.End)
+
+	var totalTriggers uint64
+
+	err := r.db.QueryRow(ctx, query).Scan(&totalTriggers)
+	if err != nil {
+		return 0, err
+	}
+
+	return totalTriggers, nil
+}
+
+func (r *ClickHouseReader) GetTriggersByInterval(ctx context.Context, ruleID string, params *model.QueryRuleStateHistory) (*v3.Series, error) {
+	step := common.MinAllowedStepInterval(params.Start, params.End)
+
+	query := fmt.Sprintf("SELECT count(*), toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL %d SECOND) as ts FROM %s.%s WHERE rule_id = '%s' AND (state_changed = true) AND (state = '%s') AND unix_milli >= %d AND unix_milli <= %d GROUP BY ts ORDER BY ts ASC",
+		step, signozHistoryDBName, ruleStateHistoryTableName, ruleID, model.StateFiring.String(), params.Start, params.End)
+
+	result, err := r.GetTimeSeriesResultV3(ctx, query)
+	if err != nil || len(result) == 0 {
+		return nil, err
+	}
+
+	return result[0], nil
+}
+
 func (r *ClickHouseReader) GetMinAndMaxTimestampForTraceID(ctx context.Context, traceID []string) (int64, int64, error) {
 	ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
 		instrumentationtypes.TelemetrySignal:  telemetrytypes.SignalTraces.StringValue(),

pkg/query-service/app/http_handler.go

@@ -15,6 +15,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/queryparser"
 
 	"log/slog"
+	"math"
 	"net/http"
 	"regexp"
 	"slices"
@@ -38,6 +39,7 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	_ "modernc.org/sqlite"
 
+	"github.com/SigNoz/signoz/pkg/contextlinks"
 	traceFunnelsModule "github.com/SigNoz/signoz/pkg/modules/tracefunnel"
 	"github.com/SigNoz/signoz/pkg/query-service/agentConf"
 	queues2 "github.com/SigNoz/signoz/pkg/query-service/app/integrations/messagingQueues/queues"
@@ -62,6 +64,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/types/opamptypes"
 	"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
 	traceFunnels "github.com/SigNoz/signoz/pkg/types/tracefunneltypes"
 
 	"github.com/SigNoz/signoz/pkg/query-service/app/integrations/messagingQueues/kafka"
@@ -367,6 +370,10 @@ func (aH *APIHandler) RegisterRoutes(router *mux.Router, am *middleware.AuthZ) {
 	router.HandleFunc("/api/v1/rules/{id}", am.EditAccess(aH.deleteRule)).Methods(http.MethodDelete)
 	router.HandleFunc("/api/v1/rules/{id}", am.EditAccess(aH.patchRule)).Methods(http.MethodPatch)
 	router.HandleFunc("/api/v1/testRule", am.EditAccess(aH.testRule)).Methods(http.MethodPost)
+	router.HandleFunc("/api/v1/rules/{id}/history/stats", am.ViewAccess(aH.getRuleStats)).Methods(http.MethodPost)
+	router.HandleFunc("/api/v1/rules/{id}/history/timeline", am.ViewAccess(aH.getRuleStateHistory)).Methods(http.MethodPost)
+	router.HandleFunc("/api/v1/rules/{id}/history/top_contributors", am.ViewAccess(aH.getRuleStateHistoryTopContributors)).Methods(http.MethodPost)
+	router.HandleFunc("/api/v1/rules/{id}/history/overall_status", am.ViewAccess(aH.getOverallStateTransitions)).Methods(http.MethodPost)
 
 	router.HandleFunc("/api/v2/variables/query", am.ViewAccess(aH.queryDashboardVarsV2)).Methods(http.MethodPost)
 
@@ -656,6 +663,217 @@ func (aH *APIHandler) testRule(w http.ResponseWriter, r *http.Request) {
 	aH.Respond(w, response)
 }
 
+func (aH *APIHandler) getRuleStats(w http.ResponseWriter, r *http.Request) {
+	ruleID := mux.Vars(r)["id"]
+	params := model.QueryRuleStateHistory{}
+	err := json.NewDecoder(r.Body).Decode(&params)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
+		return
+	}
+
+	totalCurrentTriggers, err := aH.reader.GetTotalTriggers(r.Context(), ruleID, &params)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
+		return
+	}
+	currentTriggersSeries, err := aH.reader.GetTriggersByInterval(r.Context(), ruleID, &params)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
+		return
+	}
+
+	currentAvgResolutionTime, err := aH.reader.GetAvgResolutionTime(r.Context(), ruleID, &params)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
+		return
+	}
+	currentAvgResolutionTimeSeries, err := aH.reader.GetAvgResolutionTimeByInterval(r.Context(), ruleID, &params)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
+		return
+	}
+
+	if params.End-params.Start >= 86400000 {
+		days := int64(math.Ceil(float64(params.End-params.Start) / 86400000))
+		params.Start -= days * 86400000
+		params.End -= days * 86400000
+	} else {
+		params.Start -= 86400000
+		params.End -= 86400000
+	}
+
+	totalPastTriggers, err := aH.reader.GetTotalTriggers(r.Context(), ruleID, &params)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
+		return
+	}
+	pastTriggersSeries, err := aH.reader.GetTriggersByInterval(r.Context(), ruleID, &params)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
+		return
+	}
+
+	pastAvgResolutionTime, err := aH.reader.GetAvgResolutionTime(r.Context(), ruleID, &params)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
+		return
+	}
+	pastAvgResolutionTimeSeries, err := aH.reader.GetAvgResolutionTimeByInterval(r.Context(), ruleID, &params)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
+		return
+	}
+	if math.IsNaN(currentAvgResolutionTime) || math.IsInf(currentAvgResolutionTime, 0) {
+		currentAvgResolutionTime = 0
+	}
+	if math.IsNaN(pastAvgResolutionTime) || math.IsInf(pastAvgResolutionTime, 0) {
+		pastAvgResolutionTime = 0
+	}
+
+	stats := model.Stats{
+		TotalCurrentTriggers:           totalCurrentTriggers,
+		TotalPastTriggers:              totalPastTriggers,
+		CurrentTriggersSeries:          currentTriggersSeries,
+		PastTriggersSeries:             pastTriggersSeries,
+		CurrentAvgResolutionTime:       strconv.FormatFloat(currentAvgResolutionTime, 'f', -1, 64),
+		PastAvgResolutionTime:          strconv.FormatFloat(pastAvgResolutionTime, 'f', -1, 64),
+		CurrentAvgResolutionTimeSeries: currentAvgResolutionTimeSeries,
+		PastAvgResolutionTimeSeries:    pastAvgResolutionTimeSeries,
+	}
+
+	aH.Respond(w, stats)
+}
+
+func (aH *APIHandler) getOverallStateTransitions(w http.ResponseWriter, r *http.Request) {
+	ruleID := mux.Vars(r)["id"]
+	params := model.QueryRuleStateHistory{}
+	err := json.NewDecoder(r.Body).Decode(&params)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
+		return
+	}
+
+	stateItems, err := aH.reader.GetOverallStateTransitions(r.Context(), ruleID, &params)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
+		return
+	}
+
+	aH.Respond(w, stateItems)
+}
+
+func (aH *APIHandler) getRuleStateHistory(w http.ResponseWriter, r *http.Request) {
+	idStr := mux.Vars(r)["id"]
+	id, err := valuer.NewUUID(idStr)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
+		return
+	}
+
+	params := model.QueryRuleStateHistory{}
+	err = json.NewDecoder(r.Body).Decode(&params)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
+		return
+	}
+	if err := params.Validate(); err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
+		return
+	}
+
+	res, err := aH.reader.ReadRuleStateHistoryByRuleID(r.Context(), id.StringValue(), &params)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
+		return
+	}
+
+	rule, err := aH.ruleManager.GetRule(r.Context(), id)
+	if err == nil {
+		for idx := range res.Items {
+			lbls := make(map[string]string)
+			err := json.Unmarshal([]byte(res.Items[idx].Labels), &lbls)
+			if err != nil {
+				continue
+			}
+			end := time.Unix(res.Items[idx].UnixMilli/1000, 0)
+			// why are we subtracting 3 minutes?
+			// the query range is calculated based on the rule's evalWindow and evalDelay
+			// alerts have 2 minutes delay built in, so we need to subtract that from the start time
+			// to get the correct query range
+			start := end.Add(-rule.EvalWindow.Duration() - 3*time.Minute)
+			if rule.AlertType == ruletypes.AlertTypeLogs {
+				// TODO(srikanthccv): re-visit this and support multiple queries
+				var q qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
+
+				for _, query := range rule.RuleCondition.CompositeQuery.Queries {
+					if query.Type == qbtypes.QueryTypeBuilder {
+						switch spec := query.Spec.(type) {
+						case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]:
+							q = spec
+						}
+					}
+				}
+
+				filterExpr := ""
+				if q.Filter != nil && q.Filter.Expression != "" {
+					filterExpr = q.Filter.Expression
+				}
+
+				whereClause := contextlinks.PrepareFilterExpression(lbls, filterExpr, q.GroupBy)
+
+				res.Items[idx].RelatedLogsLink = contextlinks.PrepareParamsForLogsV5(start, end, whereClause).Encode()
+			} else if rule.AlertType == ruletypes.AlertTypeTraces || rule.AlertType == ruletypes.AlertTypeAITraces {
+				// TODO(srikanthccv): re-visit this and support multiple queries
+				var q qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]
+
+				for _, query := range rule.RuleCondition.CompositeQuery.Queries {
+					if query.Type == qbtypes.QueryTypeBuilder || query.Type == qbtypes.QueryTypeBuilderAI {
+						switch spec := query.Spec.(type) {
+						case qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]:
+							q = spec
+						}
+					}
+				}
+
+				filterExpr := ""
+				if q.Filter != nil && q.Filter.Expression != "" {
+					filterExpr = q.Filter.Expression
+				}
+
+				whereClause := contextlinks.PrepareFilterExpression(lbls, filterExpr, q.GroupBy)
+				res.Items[idx].RelatedTracesLink = contextlinks.PrepareParamsForTracesV5(start, end, whereClause, rule.AlertType.BuilderQueryType()).Encode()
+			}
+		}
+	}
+
+	aH.Respond(w, res)
+}
+
+func (aH *APIHandler) getRuleStateHistoryTopContributors(w http.ResponseWriter, r *http.Request) {
+	idStr := mux.Vars(r)["id"]
+	id, err := valuer.NewUUID(idStr)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
+		return
+	}
+
+	params := model.QueryRuleStateHistory{}
+	err = json.NewDecoder(r.Body).Decode(&params)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
+		return
+	}
+
+	res, err := aH.reader.ReadRuleStateHistoryTopContributorsByRuleID(r.Context(), id.StringValue(), &params)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
+		return
+	}
+
+	aH.Respond(w, res)
+}
+
 func prepareQuery(r *http.Request) (string, error) {
 	var postData *model.DashboardVars
 

pkg/query-service/interfaces/interface.go

@@ -79,6 +79,16 @@ type Reader interface {
 
 	GetMetricMetadata(context.Context, valuer.UUID, string, string) (*v3.MetricMetadataResponse, error)
 
+	AddRuleStateHistory(ctx context.Context, ruleStateHistory []model.RuleStateHistory) error
+	GetOverallStateTransitions(ctx context.Context, ruleID string, params *model.QueryRuleStateHistory) ([]model.ReleStateItem, error)
+	ReadRuleStateHistoryByRuleID(ctx context.Context, ruleID string, params *model.QueryRuleStateHistory) (*model.RuleStateTimeline, error)
+	GetTotalTriggers(ctx context.Context, ruleID string, params *model.QueryRuleStateHistory) (uint64, error)
+	GetTriggersByInterval(ctx context.Context, ruleID string, params *model.QueryRuleStateHistory) (*v3.Series, error)
+	GetAvgResolutionTime(ctx context.Context, ruleID string, params *model.QueryRuleStateHistory) (float64, error)
+	GetAvgResolutionTimeByInterval(ctx context.Context, ruleID string, params *model.QueryRuleStateHistory) (*v3.Series, error)
+	ReadRuleStateHistoryTopContributorsByRuleID(ctx context.Context, ruleID string, params *model.QueryRuleStateHistory) ([]model.RuleStateHistoryContributor, error)
+	GetLastSavedRuleStateHistory(ctx context.Context, ruleID string) ([]model.RuleStateHistory, error)
+
 	GetMinAndMaxTimestampForTraceID(ctx context.Context, traceID []string) (int64, int64, error)
 
 	//trace

pkg/query-service/model/alerting.go

@@ -0,0 +1,202 @@
+package model
+
+import (
+	"database/sql/driver"
+	"encoding/json"
+	"fmt"
+
+	v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
+	"github.com/pkg/errors"
+)
+
+// AlertState denotes the state of an active alert.
+type AlertState int
+
+// The enum values are ordered by priority (lowest to highest).
+// When determining overall rule state, higher numeric values take precedence.
+const (
+	StateInactive AlertState = iota
+	StatePending
+	StateRecovering
+	StateFiring
+	StateNoData
+	StateDisabled
+)
+
+func (s AlertState) String() string {
+	switch s {
+	case StateInactive:
+		return "inactive"
+	case StatePending:
+		return "pending"
+	case StateFiring:
+		return "firing"
+	case StateNoData:
+		return "nodata"
+	case StateDisabled:
+		return "disabled"
+	case StateRecovering:
+		return "recovering"
+	}
+	panic(errors.Errorf("unknown alert state: %d", s))
+}
+
+func (s AlertState) MarshalJSON() ([]byte, error) {
+	return json.Marshal(s.String())
+}
+
+func (s *AlertState) UnmarshalJSON(b []byte) error {
+	var v interface{}
+	if err := json.Unmarshal(b, &v); err != nil {
+		return err
+	}
+	switch value := v.(type) {
+	case string:
+		switch value {
+		case "inactive":
+			*s = StateInactive
+		case "pending":
+			*s = StatePending
+		case "firing":
+			*s = StateFiring
+		case "nodata":
+			*s = StateNoData
+		case "disabled":
+			*s = StateDisabled
+		case "recovering":
+			*s = StateRecovering
+		default:
+			*s = StateInactive
+		}
+		return nil
+	default:
+		return errors.New("invalid alert state")
+	}
+}
+
+func (s *AlertState) Scan(value interface{}) error {
+	v, ok := value.(string)
+	if !ok {
+		return errors.New("invalid alert state")
+	}
+	switch v {
+	case "inactive":
+		*s = StateInactive
+	case "pending":
+		*s = StatePending
+	case "firing":
+		*s = StateFiring
+	case "nodata":
+		*s = StateNoData
+	case "disabled":
+		*s = StateDisabled
+	case "recovering":
+		*s = StateRecovering
+	}
+	return nil
+}
+
+func (s *AlertState) Value() (driver.Value, error) {
+	return s.String(), nil
+}
+
+type LabelsString string
+
+func (l *LabelsString) MarshalJSON() ([]byte, error) {
+	lbls := make(map[string]string)
+	err := json.Unmarshal([]byte(*l), &lbls)
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(lbls)
+}
+
+func (l *LabelsString) Scan(src interface{}) error {
+	if data, ok := src.(string); ok {
+		*l = LabelsString(data)
+	}
+	return nil
+}
+
+func (l LabelsString) String() string {
+	return string(l)
+}
+
+type RuleStateTimeline struct {
+	Items  []RuleStateHistory  `json:"items"`
+	Total  uint64              `json:"total"`
+	Labels map[string][]string `json:"labels"`
+}
+
+type RuleStateHistory struct {
+	RuleID   string `json:"ruleID" ch:"rule_id"`
+	RuleName string `json:"ruleName" ch:"rule_name"`
+	// One of ["normal", "firing"]
+	OverallState        AlertState `json:"overallState" ch:"overall_state"`
+	OverallStateChanged bool       `json:"overallStateChanged" ch:"overall_state_changed"`
+	// One of ["normal", "firing", "nodata", "muted"]
+	State        AlertState   `json:"state" ch:"state"`
+	StateChanged bool         `json:"stateChanged" ch:"state_changed"`
+	UnixMilli    int64        `json:"unixMilli" ch:"unix_milli"`
+	Labels       LabelsString `json:"labels" ch:"labels"`
+	Fingerprint  uint64       `json:"fingerprint" ch:"fingerprint"`
+	Value        float64      `json:"value" ch:"value"`
+
+	RelatedTracesLink string `json:"relatedTracesLink"`
+	RelatedLogsLink   string `json:"relatedLogsLink"`
+}
+
+type QueryRuleStateHistory struct {
+	Start   int64         `json:"start"`
+	End     int64         `json:"end"`
+	State   string        `json:"state"`
+	Filters *v3.FilterSet `json:"filters"`
+	Offset  int64         `json:"offset"`
+	Limit   int64         `json:"limit"`
+	Order   string        `json:"order"`
+}
+
+func (r *QueryRuleStateHistory) Validate() error {
+	if r.Start == 0 || r.End == 0 {
+		return fmt.Errorf("start and end are required")
+	}
+	if r.Offset < 0 || r.Limit < 0 {
+		return fmt.Errorf("offset and limit must be greater than 0")
+	}
+	if r.Order != "asc" && r.Order != "desc" {
+		return fmt.Errorf("order must be asc or desc")
+	}
+	return nil
+}
+
+type RuleStateHistoryContributor struct {
+	Fingerprint       uint64       `json:"fingerprint" ch:"fingerprint"`
+	Labels            LabelsString `json:"labels" ch:"labels"`
+	Count             uint64       `json:"count" ch:"count"`
+	RelatedTracesLink string       `json:"relatedTracesLink"`
+	RelatedLogsLink   string       `json:"relatedLogsLink"`
+}
+
+type RuleStateTransition struct {
+	RuleID         string     `json:"ruleID" ch:"rule_id"`
+	State          AlertState `json:"state" ch:"state"`
+	FiringTime     int64      `json:"firingTime" ch:"firing_time"`
+	ResolutionTime int64      `json:"resolutionTime" ch:"resolution_time"`
+}
+
+type ReleStateItem struct {
+	State AlertState `json:"state"`
+	Start int64      `json:"start"`
+	End   int64      `json:"end"`
+}
+
+type Stats struct {
+	TotalCurrentTriggers           uint64     `json:"totalCurrentTriggers"`
+	TotalPastTriggers              uint64     `json:"totalPastTriggers"`
+	CurrentTriggersSeries          *v3.Series `json:"currentTriggersSeries"`
+	PastTriggersSeries             *v3.Series `json:"pastTriggersSeries"`
+	CurrentAvgResolutionTime       string     `json:"currentAvgResolutionTime"`
+	PastAvgResolutionTime          string     `json:"pastAvgResolutionTime"`
+	CurrentAvgResolutionTimeSeries *v3.Series `json:"currentAvgResolutionTimeSeries"`
+	PastAvgResolutionTimeSeries    *v3.Series `json:"pastAvgResolutionTimeSeries"`
+}

pkg/telemetrystore/clickhousetelemetrystore/provider.go

@@ -97,8 +97,8 @@ func New(ctx context.Context, providerSettings factory.ProviderSettings, config
 	options.MaxIdleConns = config.Connection.MaxIdleConns
 	options.MaxOpenConns = config.Connection.MaxOpenConns
 	options.DialTimeout = config.Connection.DialTimeout
-	// Native flattened JSON serialization (CH 25.6+): clickhouse-go mis-decodes JSON(max_dynamic_paths=0) columns without it.
-	options.Settings["output_format_native_use_flattened_dynamic_and_json_serialization"] = 1
+	// This is to avoid the driver decoding issues with JSON columns
+	options.Settings["output_format_native_write_json_as_string"] = 1
 
 	chConn, err := clickhouse.Open(options)
 	if err != nil {
@@ -184,7 +184,7 @@ func (p *provider) Query(ctx context.Context, query string, args ...interface{})
 	}
 
 	return &rowsWithHooks{
-		Rows:    rows,
+		Rows:    telemetrystore.WrapRows(rows),
 		ctx:     ctx,
 		event:   event,
 		onClose: func() { telemetrystore.WrapAfterQuery(p.hooks, ctx, event) },

pkg/telemetrystore/rows.go

@@ -0,0 +1,39 @@
+package telemetrystore
+
+import (
+	"reflect"
+	"strings"
+
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+	"github.com/SigNoz/signoz/pkg/types/telemetrystoretypes"
+)
+
+// WrapRows reports JSONValue as the scan type of every JSON column. Nested JSON — Array(JSON),
+// Map(String, JSON) — is not covered.
+func WrapRows(rows driver.Rows) driver.Rows {
+	return &rowsWithJSONScanType{Rows: rows}
+}
+
+type rowsWithJSONScanType struct {
+	driver.Rows
+}
+
+func (r *rowsWithJSONScanType) ColumnTypes() []driver.ColumnType {
+	colTypes := r.Rows.ColumnTypes()
+	wrapped := make([]driver.ColumnType, len(colTypes))
+	for i, colType := range colTypes {
+		wrapped[i] = colType
+		if strings.HasPrefix(strings.ToUpper(colType.DatabaseTypeName()), "JSON") {
+			wrapped[i] = jsonColumnType{ColumnType: colType}
+		}
+	}
+	return wrapped
+}
+
+type jsonColumnType struct {
+	driver.ColumnType
+}
+
+func (jsonColumnType) ScanType() reflect.Type {
+	return reflect.TypeFor[telemetrystoretypes.JSONValue]()
+}

pkg/telemetrystore/telemetrystoretest/conn.go

@@ -0,0 +1,23 @@
+package telemetrystoretest
+
+import (
+	"context"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+	"github.com/SigNoz/signoz/pkg/telemetrystore"
+)
+
+// conn wraps rows the way the clickhouse provider does, so mocked JSON columns report the scan
+// type they do in production.
+type conn struct {
+	clickhouse.Conn
+}
+
+func (c conn) Query(ctx context.Context, query string, args ...any) (driver.Rows, error) {
+	rows, err := c.Conn.Query(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+	return telemetrystore.WrapRows(rows), nil
+}

pkg/telemetrystore/telemetrystoretest/provider.go

@@ -32,7 +32,7 @@ func New(_ telemetrystore.Config, matcher sqlmock.QueryMatcher) *Provider {
 
 // ClickhouseDB returns the mock Clickhouse connection.
 func (p *Provider) ClickhouseDB() clickhouse.Conn {
-	return p.clickhouseDB.(clickhouse.Conn)
+	return conn{Conn: p.clickhouseDB.(clickhouse.Conn)}
 }
 
 // Cluster returns the cluster name.

pkg/types/aiobservabilitytypes/keys.go

@@ -17,10 +17,8 @@ const (
 	GenAIUsageCacheCreationInputTokens = "gen_ai.usage.cache_creation.input_tokens"
 	GenAIUsageReasoningOutputTokens    = "gen_ai.usage.reasoning.output_tokens"
 
-	GenAIInputMessages         = "gen_ai.input.messages"
-	GenAIOutputMessages        = "gen_ai.output.messages"
-	GenAISystemInstructions    = "gen_ai.system_instructions"
-	GenAIResponseFinishReasons = "gen_ai.response.finish_reasons"
+	GenAIInputMessages  = "gen_ai.input.messages"
+	GenAIOutputMessages = "gen_ai.output.messages"
 
 	GenAIToolCallID        = "gen_ai.tool.call.id"
 	GenAIToolCallArguments = "gen_ai.tool.call.arguments"

pkg/types/aiobservabilitytypes/message.go

@@ -1,6 +1,5 @@
 // The message types mirror the OpenTelemetry GenAI message schemas, one type per schema definition:
-// https://github.com/open-telemetry/semantic-conventions-genai/blob/06ec68e722c45a7218e23ea1bc1339fe4e21ecae/model/gen-ai/gen-ai-input-messages.json
-// https://github.com/open-telemetry/semantic-conventions-genai/blob/06ec68e722c45a7218e23ea1bc1339fe4e21ecae/model/gen-ai/gen-ai-output-messages.json
+// open-telemetry/semantic-conventions-genai@06ec68e, model/gen-ai/gen-ai-{input,output}-messages.json.
 // Keys the schema does not name are dropped; the raw attribute on the span still has them.
 
 package aiobservabilitytypes

pkg/types/aiobservabilitytypes/message_format.go

@@ -4,45 +4,25 @@ import (
 	"encoding/json"
 	"fmt"
 	"regexp"
-	"sort"
-	"strconv"
 	"strings"
 
 	"github.com/SigNoz/signoz/pkg/valuer"
 )
 
 const (
-	FormatterSemconv         = "semconv"
-	FormatterOpenInference   = "openinference"
-	FormatterLangChain       = "langchain"
-	FormatterOpenRouter      = "openrouter"
-	FormatterAnthropic       = "anthropic"
-	FormatterGemini          = "gemini"
-	FormatterOpenAIChat      = "openai.chat"
-	FormatterOpenAIResponses = "openai.responses"
-	FormatterText            = "text"
-	FormatterGeneric         = "generic"
+	FormatterSemconv    = "semconv"
+	FormatterOpenAIChat = "openai.chat"
+	FormatterText       = "text"
+	FormatterGeneric    = "generic"
 )
 
 var dataURL = regexp.MustCompile(`^data:([^;,]+);base64,(.+)$`)
 
-// Provider spellings of the finish reasons; the enum values themselves pass through. tool_calls and
-// function_call are OpenAI's, completed, failed, cancelled, incomplete and max_output_tokens the Responses
-// status and incomplete reason, end_turn, stop_sequence, max_tokens, tool_use, refusal and
-// model_context_window_exceeded Anthropic's, the upper-case ones Gemini's, tool-calls and content-filter
-// the Vercel AI SDK's:
-// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L44782
-// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L69691
-// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/stop_reason.py#L5
-// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L510
-// https://github.com/vercel/ai/blob/a5e92fe3c692f8e152fe1eb9ef50a4e734ad0ffb/packages/provider/src/language-model/v2/language-model-v2-finish-reason.ts
+// Provider spellings of the finish reasons; the enum values themselves pass through.
 var finishReasonAliases = map[string]FinishReason{
-	"tool_calls": FinishReasonToolCall, "function_call": FinishReasonToolCall, "tool-calls": FinishReasonToolCall, "tool_use": FinishReasonToolCall,
-	"end_turn": FinishReasonStop, "stop_sequence": FinishReasonStop, "completed": FinishReasonStop,
-	"max_tokens": FinishReasonLength, "max_output_tokens": FinishReasonLength, "model_context_window_exceeded": FinishReasonLength,
-	"content-filter": FinishReasonContentFilter, "refusal": FinishReasonContentFilter, "safety": FinishReasonContentFilter, "recitation": FinishReasonContentFilter,
-	"blocklist": FinishReasonContentFilter, "prohibited_content": FinishReasonContentFilter, "spii": FinishReasonContentFilter, "image_safety": FinishReasonContentFilter,
-	"failed": FinishReasonError, "cancelled": FinishReasonError, "incomplete": FinishReasonError, "malformed_function_call": FinishReasonError,
+	"tool_calls": FinishReasonToolCall, "function_call": FinishReasonToolCall,
+	"end_turn": FinishReasonStop, "stop_sequence": FinishReasonStop,
+	"max_tokens": FinishReasonLength,
 }
 
 // FormattedMessages is the OTel GenAI view of one span's messages; Warnings holds what the converter could
@@ -54,43 +34,20 @@ type FormattedMessages struct {
 }
 
 // FormatMessages builds the OTel GenAI view of a span's messages from the formats SDKs write. attributes
-// holds the span's JSON attributes keyed by full dotted path. Each message value is a JSON string or a
-// structured value. An embeddings call has no messages, whatever the
-// mapper moved onto the message attributes. OpenInference's flattened messages come first,
-// then a tool call and its result, then the AI SDK's ai.* keys, then the message attributes, then
-// openllmetry's flattened prompts and completions. gen_ai.system_instructions leads the
-// input and gen_ai.response.finish_reasons overrides the finish reasons the converter found, whichever
-// converter ran.
+// is the span's JSON attribute document, nested by key segment, so an object value is whole. Each
+// message value is a JSON string or a structured value. A tool call and its result take precedence over
+// messages beside them.
 func FormatMessages(attributes map[string]any) FormattedMessages {
 	f := &formatting{attributes: attributes, out: FormattedMessages{Input: InputMessages{}, Output: OutputMessages{}, Warnings: []string{}}}
-	if isEmbeddings(attributes) {
-		return f.out
-	}
 	switch {
-	case f.openInference():
 	case attribute(f.attributes, GenAIToolCallArguments) != nil || attribute(f.attributes, GenAIToolCallResult) != nil:
 		f.tool()
-	case f.vercel():
 	case attribute(f.attributes, GenAIInputMessages) != nil || attribute(f.attributes, GenAIOutputMessages) != nil:
 		f.messages()
-	case f.traceloop():
 	}
-	f.prependSystemInstructions()
-	f.fillFinishReasons()
 	return f.out
 }
 
-// isEmbeddings matches the operation name as the semconv embeddings, openllmetry's llm.request.type
-// embedding and OpenInference's span kind EMBEDDING, copied by the mapper, or the AI SDK's ai.embed* ids.
-// https://github.com/open-telemetry/semantic-conventions-genai/blob/06ec68e722c45a7218e23ea1bc1339fe4e21ecae/model/gen-ai/registry.yaml#L658
-func isEmbeddings(attributes map[string]any) bool {
-	switch strings.ToLower(stringOf(attribute(attributes, GenAIOperationName))) {
-	case "embeddings", "embedding":
-		return true
-	}
-	return strings.HasPrefix(stringOf(attribute(attributes, vercelOperation)), "ai.embed")
-}
-
 type formatting struct {
 	attributes map[string]any
 	out        FormattedMessages
@@ -106,104 +63,32 @@ func (f *formatting) messages() {
 	if input := attribute(f.attributes, GenAIInputMessages); input != nil {
 		var msgs OutputMessages
 		msgs, inputLabel = f.side(decode(input), RoleUser)
-		f.out.Input = inputMessages(msgs)
+		for _, m := range msgs {
+			f.out.Input = append(f.out.Input, ChatMessage{Role: m.Role, Parts: m.Parts, Name: m.Name})
+		}
 	}
 	if output := attribute(f.attributes, GenAIOutputMessages); output != nil {
 		f.out.Output, outputLabel = f.side(decode(output), RoleAssistant)
-		f.completionToolCalls()
 	}
-	f.warnMixedFormats(inputLabel, outputLabel)
-}
-
-// OpenRouter's answer object or an Anthropic response beside the OpenAI chat request it answers is one
-// format, not a mix: an Anthropic request without system and without Anthropic-only blocks reads as an
-// OpenAI chat request.
-func (f *formatting) warnMixedFormats(inputLabel, outputLabel string) {
-	switch {
-	case inputLabel == "" || outputLabel == "" || outputLabel == inputLabel:
-	case inputLabel == FormatterOpenAIChat && (outputLabel == FormatterOpenRouter || outputLabel == FormatterAnthropic):
-	default:
+	if inputLabel != "" && outputLabel != "" && outputLabel != inputLabel {
 		f.warn("output formatted as %s, input as %s", outputLabel, inputLabel)
 	}
 }
 
-// prependSystemInstructions adds gen_ai.system_instructions, a list of parts, as a leading system message,
-// unless the input already starts with a system message of the same text: the OpenInference strands
-// processor keeps the attribute and inserts it as message 0 as well. A span without messages, such as
-// create_agent, stays without them.
-// https://github.com/open-telemetry/semantic-conventions-genai/blob/06ec68e722c45a7218e23ea1bc1339fe4e21ecae/model/gen-ai/gen-ai-system-instructions.json
-// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/python/instrumentation/openinference-instrumentation-strands-agents/src/openinference/instrumentation/strands_agents/processor.py#L152
-// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/python/instrumentation/openinference-instrumentation-strands-agents/src/openinference/instrumentation/strands_agents/processor.py#L293
-func (f *formatting) prependSystemInstructions() {
-	raw := attribute(f.attributes, GenAISystemInstructions)
-	if raw == nil || (len(f.out.Input) == 0 && len(f.out.Output) == 0) {
-		return
-	}
-	parts := systemParts(decode(raw))
-	if len(parts) == 0 || (len(f.out.Input) > 0 && f.out.Input[0].Role == RoleSystem && textOf(f.out.Input[0].Parts) == textOf(parts)) {
-		return
-	}
-	f.out.Input = append(InputMessages{{Role: RoleSystem, Parts: parts}}, f.out.Input...)
-}
-
-// fillFinishReasons applies gen_ai.response.finish_reasons, one entry per output message, over
-// whatever the converter found in the payload.
-// https://github.com/open-telemetry/semantic-conventions-genai/blob/06ec68e722c45a7218e23ea1bc1339fe4e21ecae/model/gen-ai/registry.yaml#L210
-func (f *formatting) fillFinishReasons() {
-	raw := attribute(f.attributes, GenAIResponseFinishReasons)
-	if raw == nil || len(f.out.Output) == 0 {
-		return
-	}
-	reasons := stringList(decode(raw))
-	if len(reasons) != len(f.out.Output) {
-		f.warn("gen_ai.response.finish_reasons has %d entries for %d output messages", len(reasons), len(f.out.Output))
-		return
-	}
-	for i, reason := range reasons {
-		if fr := finishReason(reason); fr != nil {
-			f.out.Output[i].FinishReason = fr
-		}
-	}
-}
-
 // side converts one attribute value by its shape; role is what a bare string on this side is
 // spoken as.
 func (f *formatting) side(value any, role Role) (OutputMessages, string) {
 	switch v := value.(type) {
 	case []any:
 		if first, ok := firstObject(v); ok {
-			switch {
-			case isGeminiContent(first):
-				return f.geminiContents(v), FormatterGemini
-			case first["parts"] != nil:
+			if _, ok := first["parts"]; ok {
 				return f.semconv(v), FormatterSemconv
-			case isAnthropicMessages(v):
-				return f.anthropicMessages(v), FormatterAnthropic
-			case first["role"] != nil:
+			}
+			if _, ok := first["role"]; ok {
 				return f.openAIChatMessages(v), FormatterOpenAIChat
 			}
 		}
 	case map[string]any:
-		switch {
-		case isLangChainRun(v):
-			return f.langChainRun(v), FormatterLangChain
-		case isOpenRouterCompletion(v):
-			return f.openRouterCompletion(v), FormatterOpenRouter
-		case isResponsesResponse(v):
-			return f.responsesResponse(v), FormatterOpenAIResponses
-		case isResponsesRequest(v):
-			return f.responsesRequest(v), FormatterOpenAIResponses
-		case isAnthropicResponse(v):
-			return f.anthropicResponse(v), FormatterAnthropic
-		case isAnthropicRequest(v):
-			return f.anthropicRequest(v), FormatterAnthropic
-		case isGeminiResponse(v):
-			return f.geminiResponse(v), FormatterGemini
-		case isGeminiRequest(v):
-			return f.geminiRequest(v), FormatterGemini
-		case isGeminiContent(v):
-			return OutputMessages{geminiContent(v, role)}, FormatterGemini
-		}
 		if choices, ok := v["choices"].([]any); ok {
 			return f.openAIChatResponse(choices), FormatterOpenAIChat
 		}
@@ -221,73 +106,19 @@ func (f *formatting) side(value any, role Role) (OutputMessages, string) {
 	return OutputMessages{genericMessage(value)}, FormatterGeneric
 }
 
-// attribute reads one attribute by its full dotted path from the flat JSON document, resolving the
-// driver's Variant wrappers to the plain Go types the formatter asserts on.
+// attribute walks the document along the dotted key; a nested object comes back whole. Not handled: a
+// scalar at a prefix of the path, such as gen_ai.input beside gen_ai.input.messages, which ends the
+// walk early.
 func attribute(doc map[string]any, key string) any {
-	return plainJSON(doc[key])
-}
-
-// indexed reads the list written as "<key>.<N>.<field>" keys, one object per N in numeric order. Inside an
-// entry that nested rebuilt, the list is already an object keyed by N.
-func indexed(doc map[string]any, key string) []map[string]any {
-	object, ok := attribute(doc, key).(map[string]any)
-	if !ok {
-		object = nested(doc, key+".")
-	}
-	return byIndex(object)
-}
-
-// indexedAt reads one entry of a flattened list by its index.
-func indexedAt(doc map[string]any, key string, i int) map[string]any {
-	return nested(doc, key+"."+strconv.Itoa(i)+".")
-}
-
-// nested rebuilds the keys under prefix into one object, split on their dots. When a key is both a value
-// and the parent of other keys, the object wins, so the result does not depend on map order.
-func nested(doc map[string]any, prefix string) map[string]any {
-	out := map[string]any{}
-	for key := range doc {
-		rest, ok := strings.CutPrefix(key, prefix)
-		if !ok || rest == "" {
-			continue
-		}
-		segments := strings.Split(rest, ".")
-		current := out
-		for _, segment := range segments[:len(segments)-1] {
-			child, ok := current[segment].(map[string]any)
-			if !ok {
-				child = map[string]any{}
-				current[segment] = child
-			}
-			current = child
-		}
-		leaf := segments[len(segments)-1]
-		if _, ok := current[leaf].(map[string]any); !ok {
-			current[leaf] = attribute(doc, key)
+	var current any = doc
+	for segment := range strings.SplitSeq(key, ".") {
+		object, ok := current.(map[string]any)
+		if !ok {
+			return nil
 		}
+		current = object[segment]
 	}
-	return out
-}
-
-// byIndex lists the object entries keyed by a numeric index, in index order.
-func byIndex(object map[string]any) []map[string]any {
-	indexes := make([]int, 0, len(object))
-	items := make(map[int]map[string]any, len(object))
-	for k, item := range object {
-		n, err := strconv.Atoi(k)
-		entry, isObject := item.(map[string]any)
-		if err != nil || !isObject {
-			continue
-		}
-		indexes = append(indexes, n)
-		items[n] = entry
-	}
-	sort.Ints(indexes)
-	out := make([]map[string]any, 0, len(indexes))
-	for _, n := range indexes {
-		out = append(out, items[n])
-	}
-	return out
+	return current
 }
 
 func firstObject(list []any) (map[string]any, bool) {
@@ -302,33 +133,8 @@ func genericMessage(value any) OutputMessage {
 	return OutputMessage{Parts: Parts{part(GenericPart{"type": FormatterGeneric, "content": stringOf(value)})}}
 }
 
-func inputMessages(msgs OutputMessages) InputMessages {
-	out := make(InputMessages, 0, len(msgs))
-	for _, m := range msgs {
-		out = append(out, ChatMessage{Role: m.Role, Parts: m.Parts, Name: m.Name})
-	}
-	return out
-}
-
-// foldAssistant merges runs of assistant messages into one and drops messages without parts, for
-// SDKs that write one message per output item.
-func foldAssistant(msgs OutputMessages) OutputMessages {
-	out := make(OutputMessages, 0, len(msgs))
-	for _, msg := range msgs {
-		if len(out) > 0 && msg.Role == RoleAssistant && out[len(out)-1].Role == RoleAssistant {
-			out[len(out)-1].Parts = append(out[len(out)-1].Parts, msg.Parts...)
-			continue
-		}
-		if len(msg.Parts) == 0 {
-			continue
-		}
-		out = append(out, msg)
-	}
-	return out
-}
-
 // decode parses a JSON-encoded attribute value, twice when an SDK encoded the string again.
-// Anything that is not a JSON string is returned as is.
+// Anything that is not JSON is returned as is.
 func decode(value any) any {
 	s, ok := value.(string)
 	if !ok {
@@ -348,20 +154,6 @@ func decode(value any) any {
 	return decoded
 }
 
-// plainJSON round-trips a structured value through encoding/json, turning wrapper types such as the
-// ClickHouse driver's JSON Variants into plain Go types. Values that cannot be marshaled pass through.
-func plainJSON(value any) any {
-	raw, err := json.Marshal(value)
-	if err != nil {
-		return value
-	}
-	var decoded any
-	if err := json.Unmarshal(raw, &decoded); err != nil {
-		return value
-	}
-	return decoded
-}
-
 func role(value string) Role {
 	lower := strings.ToLower(strings.TrimSpace(value))
 	switch lower {
@@ -387,23 +179,6 @@ func finishReason(value string) *FinishReason {
 	return &fr
 }
 
-// stringList reads a list of strings, or a bare string as a list of one.
-func stringList(value any) []string {
-	switch v := value.(type) {
-	case []any:
-		out := make([]string, 0, len(v))
-		for _, item := range v {
-			out = append(out, stringOf(item))
-		}
-		return out
-	case string:
-		if v != "" {
-			return []string{v}
-		}
-	}
-	return nil
-}
-
 func optionalString(value any) *string {
 	s := stringOf(value)
 	if s == "" {
@@ -420,61 +195,8 @@ func textPart(content string) TextPart {
 	return TextPart{Type: PartTypeText, Content: content}
 }
 
-// textOf joins the text parts, for comparing a system message with gen_ai.system_instructions.
-func textOf(parts Parts) string {
-	var texts []string
-	for _, p := range parts {
-		if t, ok := p.Value.(TextPart); ok {
-			texts = append(texts, t.Content)
-		}
-	}
-	return strings.Join(texts, "\n")
-}
-
-// splitToolResults moves tool_call_response parts out of a message into a tool message before it,
-// for APIs that carry tool results inside the next user turn.
-func splitToolResults(msg OutputMessage) OutputMessages {
-	results, rest := Parts{}, Parts{}
-	for _, p := range msg.Parts {
-		if _, ok := p.Value.(ToolCallResponsePart); ok {
-			results = append(results, p)
-		} else {
-			rest = append(rest, p)
-		}
-	}
-	if len(results) == 0 {
-		return OutputMessages{msg}
-	}
-	out := OutputMessages{{Role: RoleTool, Parts: results}}
-	if len(rest) > 0 {
-		msg.Parts = rest
-		out = append(out, msg)
-	}
-	return out
-}
-
-// systemParts reads a system prompt given outside the history, a list of parts or a bare string.
-func systemParts(value any) Parts {
-	switch v := value.(type) {
-	case []any:
-		parts := make(Parts, 0, len(v))
-		for _, item := range v {
-			parts = append(parts, contentPart(item))
-		}
-		return parts
-	case string:
-		if v != "" {
-			return Parts{part(textPart(v))}
-		}
-	}
-	return nil
-}
-
-// contentPart converts one content block in the OpenAI Chat Completions or Responses spelling, which
-// semconv parts may also carry; a text part in the semconv spelling has content where OpenAI has text.
-// A block of any other type is kept as sent.
-// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L53827
-// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L57589
+// contentPart converts one OpenAI content block, which semconv parts may also carry; a block of
+// any other type is kept as sent.
 func contentPart(item any) Part {
 	p, ok := item.(map[string]any)
 	if !ok {
@@ -484,14 +206,11 @@ func contentPart(item any) Part {
 		return part(GenericPart{"type": FormatterGeneric, "content": stringOf(item)})
 	}
 	switch stringOf(p["type"]) {
-	case "text", "input_text", "output_text", "summary_text":
-		if p["text"] == nil {
-			return part(textPart(stringOf(p["content"])))
-		}
+	case "text":
 		return part(textPart(stringOf(p["text"])))
 	case "refusal":
 		return part(textPart(stringOf(p["refusal"])))
-	case "image_url", "input_image":
+	case "image_url":
 		url := stringOf(p["image_url"])
 		if ref, ok := p["image_url"].(map[string]any); ok {
 			url = stringOf(ref["url"])
@@ -499,20 +218,10 @@ func contentPart(item any) Part {
 		if url != "" {
 			return part(mediaPart(url, ModalityImage))
 		}
-		if id := stringOf(p["file_id"]); id != "" {
-			return part(FilePart{Type: PartTypeFile, Modality: ModalityImage, FileID: id})
-		}
 	case "file":
 		if f, ok := filePart(p); ok {
 			return f
 		}
-	case "input_file":
-		if f, ok := filePart(map[string]any{"file": p}); ok {
-			return f
-		}
-		if url := stringOf(p["file_url"]); url != "" {
-			return part(mediaPart(url, ModalityDocument))
-		}
 	}
 	return part(GenericPart(p))
 }
@@ -530,19 +239,6 @@ func filePart(block map[string]any) (Part, bool) {
 	return Part{}, false
 }
 
-// modalityOf maps a MIME type to the schema modality, document for anything that is not media.
-func modalityOf(mimeType string) Modality {
-	switch {
-	case strings.HasPrefix(mimeType, "image/"):
-		return ModalityImage
-	case strings.HasPrefix(mimeType, "audio/"):
-		return ModalityAudio
-	case strings.HasPrefix(mimeType, "video/"):
-		return ModalityVideo
-	}
-	return ModalityDocument
-}
-
 // mediaPart places a data URL in a BlobPart, any other URL in a UriPart.
 func mediaPart(url string, modality Modality) any {
 	if m := dataURL.FindStringSubmatch(url); m != nil {

pkg/types/aiobservabilitytypes/message_format_anthropic.go

@@ -1,156 +0,0 @@
-package aiobservabilitytypes
-
-import "strings"
-
-// Format: Anthropic Messages, a request {system, messages} or a response {content, stop_reason}:
-// https://platform.claude.com/docs/en/api/messages
-// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/message_create_params.py#L181
-// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/message_param.py#L28
-// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/message.py#L91
-// Written by: gateways recording raw Anthropic bodies. OpenInference flattens these into
-// llm.*_messages, which take precedence.
-// Handled: string and block content; text, image and document sources of every type, tool_use,
-// tool_result, thinking, redacted_thinking, server_tool_use and the *_tool_result blocks. A tool_result
-// inside a user turn becomes a tool message before it.
-// Not yet: a request without system and without Anthropic-only blocks is indistinguishable from an
-// OpenAI chat request and formats as one.
-
-// Block types only the Messages API has, which tell its message list from an OpenAI chat one:
-// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/tool_use_block_param.py#L23
-// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/thinking_block_param.py#L21
-var anthropicBlockTypes = map[string]bool{
-	"tool_use": true, "tool_result": true, "thinking": true, "redacted_thinking": true, "server_tool_use": true, "document": true,
-}
-
-func isAnthropicRequest(m map[string]any) bool {
-	messages, ok := m["messages"].([]any)
-	if !ok {
-		return false
-	}
-	return m["system"] != nil || isAnthropicMessages(messages)
-}
-
-func isAnthropicResponse(m map[string]any) bool {
-	if _, ok := m["content"].([]any); !ok {
-		return false
-	}
-	_, hasStop := m["stop_reason"]
-	return hasStop || stringOf(m["type"]) == "message"
-}
-
-func isAnthropicMessages(list []any) bool {
-	for _, item := range list {
-		m, _ := item.(map[string]any)
-		blocks, _ := m["content"].([]any)
-		for _, block := range blocks {
-			b, _ := block.(map[string]any)
-			typ := stringOf(b["type"])
-			if anthropicBlockTypes[typ] || strings.HasSuffix(typ, "_tool_result") || (typ == "image" && b["source"] != nil) {
-				return true
-			}
-		}
-	}
-	return false
-}
-
-func (f *formatting) anthropicRequest(m map[string]any) OutputMessages {
-	messages, _ := m["messages"].([]any)
-	msgs := f.anthropicMessages(messages)
-	if parts := systemParts(m["system"]); len(parts) > 0 {
-		return append(OutputMessages{{Role: RoleSystem, Parts: parts}}, msgs...)
-	}
-	return msgs
-}
-
-func (f *formatting) anthropicMessages(list []any) OutputMessages {
-	out := make(OutputMessages, 0, len(list))
-	for _, item := range list {
-		m, ok := item.(map[string]any)
-		if !ok {
-			out = append(out, genericMessage(item))
-			continue
-		}
-		msg := OutputMessage{Role: role(stringOf(m["role"])), Parts: anthropicContent(m["content"])}
-		out = append(out, splitToolResults(msg)...)
-	}
-	return out
-}
-
-// anthropicResponse is one assistant message with the stop reason.
-func (f *formatting) anthropicResponse(m map[string]any) OutputMessages {
-	msg := OutputMessage{Role: role(stringOf(m["role"])), Parts: anthropicContent(m["content"]), FinishReason: finishReason(stringOf(m["stop_reason"]))}
-	if msg.Role.IsZero() {
-		msg.Role = RoleAssistant
-	}
-	return OutputMessages{msg}
-}
-
-func anthropicContent(content any) Parts {
-	parts := Parts{}
-	switch v := content.(type) {
-	case string:
-		if v != "" {
-			parts = append(parts, part(textPart(v)))
-		}
-	case []any:
-		for _, item := range v {
-			if b, ok := item.(map[string]any); ok {
-				parts = append(parts, anthropicBlock(b))
-			} else {
-				parts = append(parts, contentPart(item))
-			}
-		}
-	}
-	return parts
-}
-
-// anthropicBlock converts one content block:
-// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/tool_result_block_param.py#L29
-// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/redacted_thinking_block_param.py#L15
-// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/server_tool_use_block_param.py#L33
-// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/web_search_tool_result_block_param.py#L22
-func anthropicBlock(b map[string]any) Part {
-	typ := stringOf(b["type"])
-	switch {
-	case typ == "text":
-		return part(textPart(stringOf(b["text"])))
-	case typ == "image" || typ == "document":
-		return anthropicSource(b, typ)
-	case typ == "tool_use":
-		return part(ToolCallRequestPart{Type: PartTypeToolCall, ID: optionalString(b["id"]), Name: stringOf(b["name"]), Arguments: b["input"]})
-	case typ == "tool_result":
-		return part(ToolCallResponsePart{Type: PartTypeToolCallResponse, ID: optionalString(b["tool_use_id"]), Response: toolResponse(b["content"])})
-	case typ == "thinking":
-		return part(ReasoningPart{Type: PartTypeReasoning, Content: stringOf(b["thinking"])})
-	case typ == "redacted_thinking":
-		return part(ReasoningPart{Type: PartTypeReasoning})
-	case typ == "server_tool_use":
-		return part(ServerToolCallPart{Type: PartTypeServerToolCall, ID: optionalString(b["id"]), Name: stringOf(b["name"]), ServerToolCall: GenericServerToolCall(b)})
-	case strings.HasSuffix(typ, "_tool_result"):
-		return part(ServerToolCallResponsePart{Type: PartTypeServerToolCallResponse, ID: optionalString(b["tool_use_id"]), ServerToolCallResponse: GenericServerToolCallResponse(b)})
-	}
-	return part(GenericPart(b))
-}
-
-// anthropicSource reads the {source: {type: base64|url|file|text, ...}} of an image or document block:
-// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/image_block_param.py#L14
-// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/document_block_param.py#L16
-func anthropicSource(b map[string]any, typ string) Part {
-	source, _ := b["source"].(map[string]any)
-	mime := optionalString(source["media_type"])
-	modality := ModalityDocument
-	if typ == "image" {
-		modality = ModalityImage
-	}
-	switch stringOf(source["type"]) {
-	case "base64":
-		return part(BlobPart{Type: PartTypeBlob, Modality: modality, MimeType: mime, Content: stringOf(source["data"])})
-	case "url":
-		return part(UriPart{Type: PartTypeURI, Modality: modality, MimeType: mime, URI: stringOf(source["url"])})
-	case "file":
-		return part(FilePart{Type: PartTypeFile, Modality: modality, MimeType: mime, FileID: stringOf(source["file_id"])})
-	case "text":
-		return part(textPart(stringOf(source["data"])))
-	}
-	return part(GenericPart(b))
-}

pkg/types/aiobservabilitytypes/message_format_anthropic_test.go

@@ -1,90 +0,0 @@
-package aiobservabilitytypes
-
-import "testing"
-
-// Cases follow the Anthropic Messages API shapes; no capture in SigNoz/scripts records raw Anthropic
-// bodies.
-// https://platform.claude.com/docs/en/api/messages
-// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/message_param.py#L28
-func TestFormat_Anthropic(t *testing.T) {
-	runFormatCases(t, []formatCase{
-		{
-			// system as a string; end_turn is stop
-			// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/message_create_params.py#L181
-			// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/stop_reason.py#L5
-			name:   "SystemStringAndText_EndTurn",
-			input:  `{"model":"claude-sonnet-4-5","max_tokens":1024,"system":"You are terse.","messages":[{"role":"user","content":"Hi"}]}`,
-			output: `{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5","content":[{"type":"text","text":"Hello."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":3}}`,
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "You are terse."}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "Hi"}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "Hello."}], "finish_reason": "stop"}]`,
-		},
-		{
-			// tool_result sits inside the next user turn, with text beside it
-			// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/tool_result_block_param.py#L29
-			name:   "ToolRound_ToolResultSplitFromUserTurn",
-			input:  `{"messages":[{"role":"user","content":"Weather in Oslo?"},{"role":"assistant","content":[{"type":"text","text":"Checking."},{"type":"tool_use","id":"toolu_1","name":"get_weather","input":{"city":"Oslo"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_1","content":"{\"temp_c\":4}"},{"type":"text","text":"And Bergen?"}]}]}`,
-			output: `{"type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_2","name":"get_weather","input":{"city":"Bergen"}}],"stop_reason":"tool_use"}`,
-			wantInput: `[
-			  {"role": "user", "parts": [{"type": "text", "content": "Weather in Oslo?"}]},
-			  {"role": "assistant", "parts": [{"type": "text", "content": "Checking."}, {"type": "tool_call", "id": "toolu_1", "name": "get_weather", "arguments": {"city": "Oslo"}}]},
-			  {"role": "tool", "parts": [{"type": "tool_call_response", "id": "toolu_1", "response": {"temp_c": 4}}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "And Bergen?"}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "tool_call", "id": "toolu_2", "name": "get_weather", "arguments": {"city": "Bergen"}}], "finish_reason": "tool_call"}]`,
-		},
-		{
-			// system as text blocks; every image and document source type; thinking blocks in the history and
-			// the answer
-			// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/image_block_param.py#L14
-			// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/document_block_param.py#L16
-			// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/thinking_block_param.py#L21
-			name:   "MediaSourcesAndThinking_MaxTokens",
-			input:  `{"system":[{"type":"text","text":"Describe inputs.","cache_control":{"type":"ephemeral"}}],"messages":[{"role":"user","content":[{"type":"image","source":{"type":"base64","media_type":"image/png","data":"iVBORw0KGgo="}},{"type":"image","source":{"type":"url","url":"https://example.com/a.jpg"}},{"type":"document","source":{"type":"file","file_id":"file_1"}},{"type":"document","source":{"type":"text","media_type":"text/plain","data":"plain notes"},"title":"notes"},{"type":"text","text":"What are these?"}]},{"role":"assistant","content":[{"type":"thinking","thinking":"Two images and two documents.","signature":"sig"},{"type":"redacted_thinking","data":"xx"},{"type":"text","text":"Images and documents."}]}]}`,
-			output: `{"type":"message","role":"assistant","content":[{"type":"thinking","thinking":"Listing them.","signature":"sig"},{"type":"text","text":"A PNG, a JPEG"}],"stop_reason":"max_tokens"}`,
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "Describe inputs."}]},
-			  {"role": "user", "parts": [
-			    {"type": "blob", "modality": "image", "mime_type": "image/png", "content": "iVBORw0KGgo="},
-			    {"type": "uri", "modality": "image", "uri": "https://example.com/a.jpg"},
-			    {"type": "file", "modality": "document", "file_id": "file_1"},
-			    {"type": "text", "content": "plain notes"},
-			    {"type": "text", "content": "What are these?"}
-			  ]},
-			  {"role": "assistant", "parts": [{"type": "reasoning", "content": "Two images and two documents."}, {"type": "reasoning", "content": ""}, {"type": "text", "content": "Images and documents."}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "reasoning", "content": "Listing them."}, {"type": "text", "content": "A PNG, a JPEG"}], "finish_reason": "length"}]`,
-		},
-		{
-			// server_tool_use with its web_search_tool_result in the same message; refusal is a content filter
-			// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/server_tool_use_block_param.py#L33
-			// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/web_search_tool_result_block_param.py#L22
-			// https://github.com/anthropics/anthropic-sdk-python/blob/2a53eaa6966b04560b6a72c28d3ace4389a9ac58/src/anthropic/types/stop_reason.py#L5
-			name:      "ServerToolUseAndResult_Refusal",
-			output:    `{"type":"message","role":"assistant","content":[{"type":"server_tool_use","id":"srvtoolu_1","name":"web_search","input":{"query":"otel genai"}},{"type":"web_search_tool_result","tool_use_id":"srvtoolu_1","content":[{"type":"web_search_result","url":"https://opentelemetry.io","title":"OTel"}]},{"type":"text","text":"Found it."}],"stop_reason":"refusal"}`,
-			wantInput: `[]`,
-			wantOutput: `[{"role": "assistant", "parts": [
-			  {"type": "server_tool_call", "id": "srvtoolu_1", "name": "web_search", "server_tool_call": {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "otel genai"}}},
-			  {"type": "server_tool_call_response", "id": "srvtoolu_1", "server_tool_call_response": {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_1", "content": [{"type": "web_search_result", "url": "https://opentelemetry.io", "title": "OTel"}]}},
-			  {"type": "text", "content": "Found it."}
-			], "finish_reason": "content_filter"}]`,
-		},
-		{
-			// a request with neither system nor Anthropic-only blocks is an OpenAI chat request; the
-			// response names the format
-			name:       "PlainRequest_ResponseNamesFormat",
-			input:      `{"model":"claude-sonnet-4-5","messages":[{"role":"user","content":"Hi"}]}`,
-			output:     `{"type":"message","role":"assistant","content":[{"type":"text","text":"Hello."}],"stop_reason":"end_turn"}`,
-			wantInput:  `[{"role": "user", "parts": [{"type": "text", "content": "Hi"}]}]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "Hello."}], "finish_reason": "stop"}]`,
-		},
-		{
-			name:       "BareMessageList_ToolBlocksNameFormat",
-			input:      `[{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_1","content":[{"type":"text","text":"sunny"}]}]}]`,
-			wantInput:  `[{"role": "tool", "parts": [{"type": "tool_call_response", "id": "toolu_1", "response": "sunny"}]}]`,
-			wantOutput: `[]`,
-		},
-	})
-}

pkg/types/aiobservabilitytypes/message_format_gemini.go

@@ -1,165 +0,0 @@
-package aiobservabilitytypes
-
-// Format: Gemini generateContent, a request {contents, systemInstruction} or a response {candidates} of
-// Content {role, parts} objects, in the REST camelCase or the SDK snake_case spelling:
-// https://ai.google.dev/api/generate-content
-// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L2303
-// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L2626
-// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L8351
-// Written by: gateways recording raw Gemini bodies. OpenInference flattens these into llm.*_messages,
-// which take precedence.
-// Handled: text, thought text, inlineData, fileData, functionCall, functionResponse; a functionResponse
-// inside a user turn becomes a tool message before it. Any other part is kept as sent.
-// Not yet: a blocked prompt, promptFeedback without candidates.
-
-func isGeminiRequest(m map[string]any) bool {
-	switch m["contents"].(type) {
-	case []any, string:
-		return true
-	}
-	return false
-}
-
-func isGeminiResponse(m map[string]any) bool {
-	_, ok := m["candidates"].([]any)
-	return ok
-}
-
-// isGeminiContent tells a bare Content from a semconv message: both carry parts, only semconv parts
-// have a type.
-func isGeminiContent(m map[string]any) bool {
-	parts, ok := m["parts"].([]any)
-	if !ok {
-		return false
-	}
-	first, ok := firstObject(parts)
-	return ok && first["type"] == nil
-}
-
-// geminiRequest reads contents and systemInstruction:
-// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L6527
-func (f *formatting) geminiRequest(m map[string]any) OutputMessages {
-	var msgs OutputMessages
-	if system := geminiSystemInstruction(objectAt(m, "systemInstruction", "system_instruction")); system != nil {
-		msgs = append(msgs, *system)
-	}
-	switch contents := m["contents"].(type) {
-	case string:
-		msgs = append(msgs, OutputMessage{Role: RoleUser, Parts: Parts{part(textPart(contents))}})
-	case []any:
-		msgs = append(msgs, f.geminiContents(contents)...)
-	}
-	return msgs
-}
-
-// geminiResponse is one message per candidate with its finish reason.
-func (f *formatting) geminiResponse(m map[string]any) OutputMessages {
-	candidates, _ := m["candidates"].([]any)
-	out := make(OutputMessages, 0, len(candidates))
-	for _, item := range candidates {
-		candidate, ok := item.(map[string]any)
-		if !ok {
-			out = append(out, genericMessage(item))
-			continue
-		}
-		content, _ := candidate["content"].(map[string]any)
-		msg := geminiContent(content, RoleAssistant)
-		msg.FinishReason = finishReason(stringOf(objectAt(candidate, "finishReason", "finish_reason")))
-		out = append(out, msg)
-	}
-	return out
-}
-
-// geminiSystemInstruction accepts a string, a Content or a list of parts.
-func geminiSystemInstruction(value any) *OutputMessage {
-	var parts Parts
-	switch v := value.(type) {
-	case map[string]any:
-		parts = geminiContent(v, RoleSystem).Parts
-	case []any:
-		parts = geminiParts(v)
-	default:
-		parts = systemParts(value)
-	}
-	if len(parts) == 0 {
-		return nil
-	}
-	return &OutputMessage{Role: RoleSystem, Parts: parts}
-}
-
-func (f *formatting) geminiContents(list []any) OutputMessages {
-	out := make(OutputMessages, 0, len(list))
-	for _, item := range list {
-		switch v := item.(type) {
-		case string:
-			out = append(out, OutputMessage{Role: RoleUser, Parts: Parts{part(textPart(v))}})
-		case map[string]any:
-			out = append(out, splitToolResults(geminiContent(v, RoleUser))...)
-		default:
-			out = append(out, genericMessage(item))
-		}
-	}
-	return out
-}
-
-func geminiContent(c map[string]any, fallback Role) OutputMessage {
-	msg := OutputMessage{Role: role(stringOf(c["role"])), Parts: Parts{}}
-	if msg.Role.IsZero() {
-		msg.Role = fallback
-	}
-	parts, _ := c["parts"].([]any)
-	msg.Parts = geminiParts(parts)
-	return msg
-}
-
-func geminiParts(list []any) Parts {
-	parts := Parts{}
-	for _, item := range list {
-		if p, ok := item.(map[string]any); ok {
-			parts = append(parts, geminiPart(p))
-		} else {
-			parts = append(parts, contentPart(item))
-		}
-	}
-	return parts
-}
-
-// geminiPart converts one Part by the one field it carries:
-// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L2324
-// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L2112
-// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L1731
-// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L1830
-// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L2032
-func geminiPart(p map[string]any) Part {
-	if text, ok := p["text"].(string); ok {
-		if thought, _ := p["thought"].(bool); thought {
-			return part(ReasoningPart{Type: PartTypeReasoning, Content: text})
-		}
-		return part(textPart(text))
-	}
-	if data, ok := objectAt(p, "inlineData", "inline_data").(map[string]any); ok {
-		mime := stringOf(objectAt(data, "mimeType", "mime_type"))
-		return part(BlobPart{Type: PartTypeBlob, Modality: modalityOf(mime), MimeType: optionalString(mime), Content: stringOf(data["data"])})
-	}
-	if file, ok := objectAt(p, "fileData", "file_data").(map[string]any); ok {
-		mime := stringOf(objectAt(file, "mimeType", "mime_type"))
-		return part(UriPart{Type: PartTypeURI, Modality: modalityOf(mime), MimeType: optionalString(mime), URI: stringOf(objectAt(file, "fileUri", "file_uri"))})
-	}
-	if call, ok := objectAt(p, "functionCall", "function_call").(map[string]any); ok {
-		return part(ToolCallRequestPart{Type: PartTypeToolCall, ID: optionalString(call["id"]), Name: stringOf(call["name"]), Arguments: call["args"]})
-	}
-	if response, ok := objectAt(p, "functionResponse", "function_response").(map[string]any); ok {
-		return part(ToolCallResponsePart{Type: PartTypeToolCallResponse, ID: optionalString(response["id"]), Response: response["response"]})
-	}
-	return part(GenericPart(p))
-}
-
-// objectAt reads the first of the key spellings that is set.
-func objectAt(m map[string]any, keys ...string) any {
-	for _, key := range keys {
-		if v, ok := m[key]; ok && v != nil {
-			return v
-		}
-	}
-	return nil
-}

pkg/types/aiobservabilitytypes/message_format_gemini_test.go

@@ -1,75 +0,0 @@
-package aiobservabilitytypes
-
-import "testing"
-
-// Cases follow the Gemini generateContent shapes, REST camelCase and SDK snake_case; no capture in
-// SigNoz/scripts records raw Gemini bodies.
-// https://ai.google.dev/api/generate-content
-// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L2303
-func TestFormat_Gemini(t *testing.T) {
-	runFormatCases(t, []formatCase{
-		{
-			// systemInstruction as a Content; STOP is stop
-			// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L6527
-			// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L510
-			name:   "SystemInstructionContent_CandidateText_Stop",
-			input:  `{"contents":[{"role":"user","parts":[{"text":"Hi"}]}],"systemInstruction":{"parts":[{"text":"You are terse."}]},"generationConfig":{"temperature":0}}`,
-			output: `{"candidates":[{"content":{"parts":[{"text":"Hello."}],"role":"model"},"finishReason":"STOP","index":0}],"usageMetadata":{"promptTokenCount":5,"candidatesTokenCount":2},"modelVersion":"gemini-2.5-flash"}`,
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "You are terse."}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "Hi"}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "Hello."}], "finish_reason": "stop"}]`,
-		},
-		{
-			// SDK snake_case spelling; the function response sits in the next user turn
-			// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L1830
-			// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L2032
-			name:   "FunctionRoundSnakeCase_ResponseSplitFromUserTurn",
-			input:  `{"model":"gemini-2.5-pro","contents":[{"role":"user","parts":[{"text":"Weather in Oslo?"}]},{"role":"model","parts":[{"function_call":{"name":"get_weather","args":{"city":"Oslo"}}}]},{"role":"user","parts":[{"function_response":{"name":"get_weather","response":{"temp_c":4}}},{"text":"And Bergen?"}]}],"system_instruction":"Use tools."}`,
-			output: `{"candidates":[{"content":{"role":"model","parts":[{"function_call":{"id":"fc_1","name":"get_weather","args":{"city":"Bergen"}}}]},"finish_reason":"STOP"}]}`,
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "Use tools."}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "Weather in Oslo?"}]},
-			  {"role": "assistant", "parts": [{"type": "tool_call", "name": "get_weather", "arguments": {"city": "Oslo"}}]},
-			  {"role": "tool", "parts": [{"type": "tool_call_response", "response": {"temp_c": 4}}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "And Bergen?"}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "tool_call", "id": "fc_1", "name": "get_weather", "arguments": {"city": "Bergen"}}], "finish_reason": "stop"}]`,
-		},
-		{
-			// inline and file data, a thought part, an executable code part kept as sent; two candidates, the
-			// second stopped for SAFETY
-			// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L2112
-			// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L1731
-			// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L2344
-			// https://github.com/googleapis/python-genai/blob/78f6d3412caaaed44f99ce57be2d16d0b8c407cb/google/genai/types.py#L510
-			name:   "MediaAndThought_TwoCandidates",
-			input:  `{"contents":[{"parts":[{"inlineData":{"mimeType":"image/png","data":"iVBORw0KGgo="}},{"fileData":{"mimeType":"video/mp4","fileUri":"gs://bucket/a.mp4"}},{"text":"Describe."}]}]}`,
-			output: `{"candidates":[{"content":{"role":"model","parts":[{"text":"Looking.","thought":true},{"executableCode":{"language":"PYTHON","code":"print(1)"}},{"text":"A logo and a clip."}]},"finishReason":"MAX_TOKENS"},{"content":{"role":"model","parts":[]},"finishReason":"SAFETY"}]}`,
-			wantInput: `[{"role": "user", "parts": [
-			  {"type": "blob", "modality": "image", "mime_type": "image/png", "content": "iVBORw0KGgo="},
-			  {"type": "uri", "modality": "video", "mime_type": "video/mp4", "uri": "gs://bucket/a.mp4"},
-			  {"type": "text", "content": "Describe."}
-			]}]`,
-			wantOutput: `[
-			  {"role": "assistant", "parts": [{"type": "reasoning", "content": "Looking."}, {"executableCode": {"language": "PYTHON", "code": "print(1)"}}, {"type": "text", "content": "A logo and a clip."}], "finish_reason": "length"},
-			  {"role": "assistant", "parts": [], "finish_reason": "content_filter"}
-			]`,
-		},
-		{
-			name:       "StringContents_UserText",
-			input:      `{"model":"gemini-2.5-flash","contents":"Hi"}`,
-			output:     `{"role":"model","parts":[{"text":"Hello."}]}`,
-			wantInput:  `[{"role": "user", "parts": [{"type": "text", "content": "Hi"}]}]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "Hello."}]}]`,
-		},
-		{
-			// a bare Content list is told from semconv messages by its untyped parts
-			name:       "BareContentList_NotSemconv",
-			input:      `[{"role":"user","parts":[{"text":"Hi"}]}]`,
-			wantInput:  `[{"role": "user", "parts": [{"type": "text", "content": "Hi"}]}]`,
-			wantOutput: `[]`,
-		},
-	})
-}

pkg/types/aiobservabilitytypes/message_format_langchain.go

@@ -1,200 +0,0 @@
-package aiobservabilitytypes
-
-import "github.com/SigNoz/signoz/pkg/valuer"
-
-// Format: LangChain's serialised run inputs and outputs, {messages: [[Serializable]]} and
-// {generations: [[{text, generation_info, message}]]}, each message {lc: 1, type: "constructor",
-// id: [..., class], kwargs: {content, tool_calls, tool_call_id, name, ...}}:
-// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/core/langchain_core/load/serializable.py#L297
-// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/core/langchain_core/outputs/llm_result.py#L23
-// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/core/langchain_core/messages/ai.py#L170
-// Written by: the LangSmith OTel exporter as gen_ai.prompt and gen_ai.completion, which the span mapper
-// moves onto gen_ai.input.messages and gen_ai.output.messages:
-// https://github.com/langchain-ai/langsmith-sdk/blob/bf325f3a0d9f415096a4632099b7faff355cde53/python/langsmith/_internal/otel/_otel_exporter.py#L761
-// https://github.com/langchain-ai/langsmith-sdk/blob/bf325f3a0d9f415096a4632099b7faff355cde53/python/langsmith/_internal/otel/_otel_exporter.py#L843
-// The envelope and the message classes are langchain_core's and shared by every partner package; the
-// partners differ in the key that carries the finish reason and in the content blocks they keep.
-// Handled: string content, langchain_core's standard content blocks and the OpenAI block spellings,
-// tool_calls, tool messages, the finish reason of a generation.
-// Not yet: the provider-native thinking blocks anthropic, bedrock and google keep in the content, kept
-// as sent:
-// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/partners/anthropic/langchain_anthropic/chat_models.py#L866
-// https://github.com/langchain-ai/langchain-aws/blob/910b79b9787eee773cde65bdd9e94266d6cf6f6c/libs/aws/langchain_aws/chat_models/bedrock_converse.py#L401
-// https://github.com/langchain-ai/langchain-google/blob/9f72e9a344235a488396ac2c62d360ceb2b1c81c/libs/genai/langchain_google_genai/chat_models.py#L165
-
-// LangChain message classes to roles; ChatMessage carries its own role in kwargs.
-var langChainRoles = map[string]Role{
-	"SystemMessage": RoleSystem, "SystemMessageChunk": RoleSystem,
-	"HumanMessage": RoleUser, "HumanMessageChunk": RoleUser,
-	"AIMessage": RoleAssistant, "AIMessageChunk": RoleAssistant,
-	"ToolMessage": RoleTool, "ToolMessageChunk": RoleTool, "FunctionMessage": RoleTool, "FunctionMessageChunk": RoleTool,
-}
-
-// isLangChainRun looks for the generations list or a messages batch of serialised messages.
-func isLangChainRun(m map[string]any) bool {
-	if _, ok := m["generations"].([]any); ok {
-		return true
-	}
-	batches, ok := m["messages"].([]any)
-	if !ok || len(batches) == 0 {
-		return false
-	}
-	first, ok := batches[0].([]any)
-	if !ok {
-		return false
-	}
-	msg, ok := firstObject(first)
-	return ok && msg["lc"] != nil
-}
-
-// langChainRun converts the run inputs, every batch of messages in order, or the run outputs, one
-// message per generation.
-func (f *formatting) langChainRun(m map[string]any) OutputMessages {
-	var out OutputMessages
-	if batches, ok := m["messages"].([]any); ok {
-		for _, batch := range batches {
-			if list, ok := batch.([]any); ok {
-				out = append(out, f.langChainMessages(list)...)
-			}
-		}
-		return out
-	}
-	batches, _ := m["generations"].([]any)
-	for _, batch := range batches {
-		list, _ := batch.([]any)
-		for _, item := range list {
-			generation, ok := item.(map[string]any)
-			if !ok {
-				out = append(out, genericMessage(item))
-				continue
-			}
-			msg := OutputMessage{Role: RoleAssistant, Parts: Parts{}}
-			if message, ok := generation["message"].(map[string]any); ok {
-				msg = f.langChainMessage(message)
-			} else if text := stringOf(generation["text"]); text != "" {
-				msg.Parts = Parts{part(textPart(text))}
-			}
-			if info, ok := generation["generation_info"].(map[string]any); ok && msg.FinishReason == nil {
-				msg.FinishReason = finishReason(stringOf(info["finish_reason"]))
-			}
-			out = append(out, msg)
-		}
-	}
-	return out
-}
-
-func (f *formatting) langChainMessages(list []any) OutputMessages {
-	out := make(OutputMessages, 0, len(list))
-	for _, item := range list {
-		m, ok := item.(map[string]any)
-		if !ok {
-			out = append(out, genericMessage(item))
-			continue
-		}
-		out = append(out, f.langChainMessage(m))
-	}
-	return out
-}
-
-// langChainMessage converts one serialised message. A tool_call or tool_use block in the content repeats
-// a call that tool_calls already carries, langchain_core adds the former from the latter and
-// langchain-anthropic keeps the provider's block beside it:
-// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/core/langchain_core/messages/ai.py#L274
-// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/partners/anthropic/langchain_anthropic/chat_models.py#L2571
-// A failed call leaves the error body in response_metadata.
-func (f *formatting) langChainMessage(m map[string]any) OutputMessage {
-	kwargs, _ := m["kwargs"].(map[string]any)
-	msg := OutputMessage{Role: langChainRole(m, kwargs), Parts: Parts{}, Name: optionalString(kwargs["name"])}
-	if id := stringOf(kwargs["tool_call_id"]); msg.Role == RoleTool && id != "" {
-		msg.Parts = Parts{part(ToolCallResponsePart{Type: PartTypeToolCallResponse, ID: &id, Response: toolResponse(kwargs["content"])})}
-		return msg
-	}
-	calls, _ := kwargs["tool_calls"].([]any)
-	switch content := kwargs["content"].(type) {
-	case string:
-		if content != "" {
-			msg.Parts = append(msg.Parts, part(textPart(content)))
-		}
-	case []any:
-		for _, item := range content {
-			if block, ok := item.(map[string]any); ok && len(calls) > 0 && (block["type"] == "tool_call" || block["type"] == "tool_use") {
-				continue
-			}
-			msg.Parts = append(msg.Parts, langChainBlock(item))
-		}
-	}
-	for _, item := range calls {
-		if call, ok := item.(map[string]any); ok {
-			msg.Parts = append(msg.Parts, part(ToolCallRequestPart{Type: PartTypeToolCall, Name: stringOf(call["name"]), ID: optionalString(call["id"]), Arguments: decode(call["args"])}))
-		}
-	}
-	if metadata, ok := kwargs["response_metadata"].(map[string]any); ok {
-		msg.FinishReason = langChainFinishReason(metadata)
-		if body, ok := metadata["body"].(map[string]any); ok && body["error"] != nil {
-			msg.FinishReason = finishReason("error")
-		}
-	}
-	return msg
-}
-
-// langChainFinishReason reads the key each partner writes into response_metadata: finish_reason for
-// most, stop_reason for anthropic, stopReason for bedrock, done_reason for ollama:
-// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/partners/anthropic/langchain_anthropic/chat_models.py#L2510
-// https://github.com/langchain-ai/langchain-aws/blob/910b79b9787eee773cde65bdd9e94266d6cf6f6c/libs/aws/langchain_aws/chat_models/bedrock_converse.py#L265
-// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/partners/ollama/langchain_ollama/chat_models.py#L24
-func langChainFinishReason(metadata map[string]any) *FinishReason {
-	for _, key := range []string{"finish_reason", "stop_reason", "stopReason", "done_reason"} {
-		if fr := finishReason(stringOf(metadata[key])); fr != nil {
-			return fr
-		}
-	}
-	return nil
-}
-
-// langChainBlock converts one content block: langchain_core's standard reasoning and media blocks, else
-// the OpenAI spellings contentPart knows:
-// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/core/langchain_core/messages/content.py#L456
-// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/core/langchain_core/messages/content.py#L498
-func langChainBlock(item any) Part {
-	block, ok := item.(map[string]any)
-	if !ok {
-		return contentPart(item)
-	}
-	typ := stringOf(block["type"])
-	switch typ {
-	case "reasoning":
-		return part(ReasoningPart{Type: PartTypeReasoning, Content: stringOf(block["reasoning"])})
-	case "image", "audio", "video", "file":
-		modality := Modality{valuer.NewString(typ)}
-		if typ == "file" {
-			modality = ModalityDocument
-		}
-		mime := optionalString(block["mime_type"])
-		switch {
-		case stringOf(block["url"]) != "":
-			return part(mediaPart(stringOf(block["url"]), modality))
-		case stringOf(block["base64"]) != "":
-			return part(BlobPart{Type: PartTypeBlob, Modality: modality, MimeType: mime, Content: stringOf(block["base64"])})
-		case stringOf(block["file_id"]) != "":
-			return part(FilePart{Type: PartTypeFile, Modality: modality, MimeType: mime, FileID: stringOf(block["file_id"])})
-		}
-	}
-	return contentPart(item)
-}
-
-// langChainRole reads the message class from the serialised id path, falling back to the type
-// alias in kwargs:
-// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/core/langchain_core/messages/chat.py#L18
-// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/core/langchain_core/messages/human.py#L29
-func langChainRole(m, kwargs map[string]any) Role {
-	if id, ok := m["id"].([]any); ok && len(id) > 0 {
-		class := stringOf(id[len(id)-1])
-		if r, ok := langChainRoles[class]; ok {
-			return r
-		}
-		if class == "ChatMessage" || class == "ChatMessageChunk" {
-			return role(stringOf(kwargs["role"]))
-		}
-	}
-	return role(stringOf(kwargs["type"]))
-}

pkg/types/aiobservabilitytypes/message_format_langchain_test.go

@@ -1,77 +0,0 @@
-package aiobservabilitytypes
-
-import "testing"
-
-// Captured cases are gen_ai.input.messages and gen_ai.output.messages as the collector stored them for
-// the SigNoz/scripts langsmith capture, the LangSmith OTel exporter's serialised run inputs and outputs;
-// the others follow langchain_core and the partner packages at the linked lines.
-func TestFormat_LangChain(t *testing.T) {
-	runFormatCases(t, []formatCase{
-		{
-			// langsmith capture, ChatOpenAI span after the tool ran
-			name:   "SerialisedMessages_RolesFromClasses_ToolCallAndToolMessage",
-			input:  `{"messages":[[{"lc":1,"type":"constructor","id":["langchain","schema","messages","SystemMessage"],"kwargs":{"content":"You are a weather assistant. Call get_weather once per city.","type":"system"}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","HumanMessage"],"kwargs":{"content":"What's the weather in Bengaluru?","type":"human"}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"","additional_kwargs":{"refusal":null},"response_metadata":{"finish_reason":"tool_calls","logprobs":null},"type":"ai","id":"lc_run--019e844a","tool_calls":[{"name":"get_weather","args":{"city":"Bengaluru"},"id":"call_nbrbkc5fom95q8uMA2bCvH6s","type":"tool_call"}],"invalid_tool_calls":[]}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","ToolMessage"],"kwargs":{"content":"{\"city\": \"Bengaluru\", \"temp_c\": 18, \"summary\": \"Clear\"}","type":"tool","tool_call_id":"call_nbrbkc5fom95q8uMA2bCvH6s","status":"success"}}]]}`,
-			output: `{"generations":[[{"text":"The weather in Bengaluru is clear, 18°C.","generation_info":{"finish_reason":"stop","logprobs":null},"type":"ChatGeneration","message":{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"The weather in Bengaluru is clear, 18°C.","additional_kwargs":{"refusal":null},"response_metadata":{"finish_reason":"stop","model_name":"gpt-4o-mini-2024-07-18"},"type":"ai","tool_calls":[],"invalid_tool_calls":[]}}}]],"llm_output":{"model_name":"gpt-4o-mini-2024-07-18"},"run":null,"type":"LLMResult"}`,
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "You are a weather assistant. Call get_weather once per city."}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "What's the weather in Bengaluru?"}]},
-			  {"role": "assistant", "parts": [{"type": "tool_call", "name": "get_weather", "id": "call_nbrbkc5fom95q8uMA2bCvH6s", "arguments": {"city": "Bengaluru"}}]},
-			  {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_nbrbkc5fom95q8uMA2bCvH6s", "response": {"city": "Bengaluru", "temp_c": 18, "summary": "Clear"}}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "The weather in Bengaluru is clear, 18°C."}], "finish_reason": "stop"}]`,
-		},
-		{
-			// langsmith capture, ChatOpenAI span answering with a tool call: the generation text is empty and
-			// the finish reason sits in generation_info
-			// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/core/langchain_core/outputs/generation.py#L28
-			name:       "GenerationWithToolCalls_ToolCallPart_FinishFromGenerationInfo",
-			output:     `{"generations":[[{"text":"","generation_info":{"finish_reason":"tool_calls"},"type":"ChatGeneration","message":{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"","response_metadata":{},"type":"ai","tool_calls":[{"name":"get_weather","args":{"city":"Bengaluru"},"id":"call_1","type":"tool_call"}]}}}]]}`,
-			wantInput:  `[]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "get_weather", "id": "call_1", "arguments": {"city": "Bengaluru"}}], "finish_reason": "tool_call"}]`,
-		},
-		{
-			// langsmith capture, ChatOpenAI span of a failed call: the error body sits in response_metadata
-			name:       "GenerationWithErrorBody_EmptyMessage_FinishError",
-			output:     `{"generations":[[{"text":"","generation_info":null,"type":"ChatGeneration","message":{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"","response_metadata":{"body":{"error":{"message":"Incorrect API key provided","type":"invalid_request_error","code":"invalid_api_key"},"status":401},"headers":{}},"type":"ai"}}}]]}`,
-			wantInput:  `[]`,
-			wantOutput: `[{"role": "assistant", "parts": [], "finish_reason": "error"}]`,
-		},
-		{
-			// a ChatMessage carries its role in kwargs; an AIMessage from langchain-anthropic keeps the tool_use
-			// block in its content beside tool_calls
-			// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/core/langchain_core/messages/chat.py#L18
-			// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/partners/anthropic/langchain_anthropic/chat_models.py#L2571
-			name:  "ChatMessageRole_ToolUseBlockBesideToolCalls_OneToolCallPart",
-			input: `{"messages":[[{"lc":1,"type":"constructor","id":["langchain","schema","messages","ChatMessage"],"kwargs":{"role":"developer","content":[{"type":"text","text":"Be brief."}]}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":[{"type":"text","text":"Calling."},{"type":"tool_use","id":"toolu_1","name":"get_weather","input":{"city":"Paris"}}],"tool_calls":[{"name":"get_weather","args":{"city":"Paris"},"id":"toolu_1"}]}}]]}`,
-			wantInput: `[
-			  {"role": "developer", "parts": [{"type": "text", "content": "Be brief."}]},
-			  {"role": "assistant", "parts": [{"type": "text", "content": "Calling."}, {"type": "tool_call", "name": "get_weather", "id": "toolu_1", "arguments": {"city": "Paris"}}]}
-			]`,
-			wantOutput: `[]`,
-		},
-		{
-			// the finish reason under the key each partner writes: anthropic stop_reason, ollama done_reason
-			// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/partners/anthropic/langchain_anthropic/chat_models.py#L2510
-			// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/partners/ollama/langchain_ollama/chat_models.py#L24
-			name:       "PartnerFinishReasonKeys_StopReasonAndDoneReason",
-			output:     `{"generations":[[{"text":"Hi.","message":{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"Hi.","response_metadata":{"stop_reason":"end_turn","model":"claude-sonnet-4-5"}}}}],[{"text":"Hel","message":{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"Hel","response_metadata":{"model":"llama3","done_reason":"length"}}}}]]}`,
-			wantInput:  `[]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "Hi."}], "finish_reason": "stop"}, {"role": "assistant", "parts": [{"type": "text", "content": "Hel"}], "finish_reason": "length"}]`,
-		},
-		{
-			// langchain_core's standard content blocks, as partners write them under output_version v1
-			// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/core/langchain_core/messages/content.py#L456
-			// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/core/langchain_core/messages/content.py#L498
-			// https://github.com/langchain-ai/langchain/blob/84ba23c414eca6bf6f6ebd1394a3006378113540/libs/core/langchain_core/messages/content.py#L721
-			name:      "StandardContentBlocks_ReasoningAndMediaParts",
-			output:    `{"generations":[[{"text":"It is a duck.","message":{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":[{"type":"reasoning","reasoning":"Looks like a bird."},{"type":"text","text":"It is a duck."},{"type":"image","base64":"iVBORw0KGgo=","mime_type":"image/png"},{"type":"file","url":"https://example.com/a.pdf","mime_type":"application/pdf"}],"response_metadata":{"finish_reason":"stop","output_version":"v1"}}}}]]}`,
-			wantInput: `[]`,
-			wantOutput: `[{"role": "assistant", "parts": [
-			  {"type": "reasoning", "content": "Looks like a bird."},
-			  {"type": "text", "content": "It is a duck."},
-			  {"type": "blob", "modality": "image", "mime_type": "image/png", "content": "iVBORw0KGgo="},
-			  {"type": "uri", "modality": "document", "uri": "https://example.com/a.pdf"}
-			], "finish_reason": "stop"}]`,
-		},
-	})
-}

pkg/types/aiobservabilitytypes/message_format_openai_chat_test.go

@@ -167,8 +167,7 @@ func TestFormat_OpenAIChat(t *testing.T) {
 			]`,
 		},
 		{
-			// ChatCompletionRequestMessageContentPartFile {file: {file_id}}, as OpenInference records the request in input.value:
-			// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L42650
+			// openinference capture, input.value of the ChatCompletion span
 			name:   "FileByID_FilePart",
 			input:  `{"model": "gpt-4o-mini", "messages": [{"role": "user", "content": [{"type": "text", "text": "Summarise this document in one sentence."}, {"type": "file", "file": {"file_id": "file-6F2ksmvXxt4VdoqmHRw6kL"}}]}], "max_tokens": 60}`,
 			output: `{"id":"chatcmpl-DktFileCapture0001","object":"chat.completion","model":"gpt-4o-mini-2024-07-18","choices":[{"index":0,"finish_reason":"stop","logprobs":null,"message":{"role":"assistant","content":"The document is a short PDF about dragons.","refusal":null,"annotations":[]}}]}`,
@@ -190,8 +189,7 @@ func TestFormat_OpenAIChat(t *testing.T) {
 			]`,
 		},
 		{
-			// ChatCompletionRequestMessageContentPartFile {file: {filename, file_data}}, file_data a data URL:
-			// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L42650
+			// openinference capture, input.value of the ChatCompletion span
 			name:  "FileInline_BlobPart",
 			input: `{"model": "gpt-4o-mini", "messages": [{"role": "user", "content": [{"type": "text", "text": "Summarise this document in one sentence."}, {"type": "file", "file": {"filename": "draconomicon.pdf", "file_data": "data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iajw8L1R5cGUvQ2F0YWxvZy9QYWdlcyAyIDAgUj4+ZW5kb2JqCjIgMCBvYmo8PC9UeXBlL1BhZ2VzL0tpZHNbXS9Db3VudCAwPj5lbmRvYmoKdHJhaWxlcjw8L1Jvb3QgMSAwIFI+PgolJUVPRgo="}}]}], "max_tokens": 60}`,
 			wantInput: `[

pkg/types/aiobservabilitytypes/message_format_openai_responses.go

@@ -1,136 +0,0 @@
-package aiobservabilitytypes
-
-import "strings"
-
-// Format: OpenAI Responses, a request {input, instructions} or a response {output, status} of items:
-// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L47597
-// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L69691
-// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L53834
-// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L57596
-// Written by: gateways recording raw Responses bodies, and the raw calls OpenInference records for the
-// Responses API and the OpenAI Agents SDK when its flattened messages are absent.
-// Handled: message items with input_text, output_text, refusal, input_image and input_file content;
-// function_call and custom_tool_call with their outputs; reasoning summaries; compaction; every other
-// *_call item as a server tool call and *_output item as its response; the status and incomplete_details
-// of a response.
-// Dropped: reasoning encrypted_content, item ids. Not yet: input_audio content.
-
-// isResponsesRequest tells a Responses request from an embeddings one, which also carries input and
-// model: items are objects, and only Responses takes instructions or tools.
-func isResponsesRequest(m map[string]any) bool {
-	if m["input"] == nil {
-		return false
-	}
-	if m["instructions"] != nil || m["tools"] != nil || m["previous_response_id"] != nil {
-		return true
-	}
-	items, ok := m["input"].([]any)
-	if !ok {
-		return false
-	}
-	_, isItem := firstObject(items)
-	return isItem
-}
-
-func isResponsesResponse(m map[string]any) bool {
-	if _, ok := m["output"].([]any); !ok {
-		return false
-	}
-	return stringOf(m["object"]) == "response" || m["status"] != nil
-}
-
-func (f *formatting) responsesRequest(m map[string]any) OutputMessages {
-	var msgs OutputMessages
-	if parts := systemParts(m["instructions"]); len(parts) > 0 {
-		msgs = append(msgs, OutputMessage{Role: RoleSystem, Parts: parts})
-	}
-	switch input := m["input"].(type) {
-	case string:
-		msgs = append(msgs, OutputMessage{Role: RoleUser, Parts: Parts{part(textPart(input))}})
-	case []any:
-		msgs = append(msgs, foldAssistant(f.responsesItems(input))...)
-	}
-	return msgs
-}
-
-// responsesResponse folds the output items into one message: the API returns a single choice. The
-// finish reason is the incomplete_details reason, else the status:
-// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L69763
-func (f *formatting) responsesResponse(m map[string]any) OutputMessages {
-	output, _ := m["output"].([]any)
-	msgs := foldAssistant(f.responsesItems(output))
-	if len(msgs) == 0 {
-		return msgs
-	}
-	last := &msgs[len(msgs)-1]
-	if details, ok := m["incomplete_details"].(map[string]any); ok {
-		last.FinishReason = finishReason(stringOf(details["reason"]))
-	} else if status := stringOf(m["status"]); status != "" {
-		last.FinishReason = finishReason(status)
-	}
-	return msgs
-}
-
-// responsesItems converts items one message each; the caller folds assistant runs.
-func (f *formatting) responsesItems(items []any) OutputMessages {
-	msgs := make(OutputMessages, 0, len(items))
-	for _, item := range items {
-		m, ok := item.(map[string]any)
-		if !ok {
-			if s, ok := item.(string); ok {
-				msgs = append(msgs, OutputMessage{Role: RoleUser, Parts: Parts{part(textPart(s))}})
-				continue
-			}
-			msgs = append(msgs, genericMessage(item))
-			continue
-		}
-		typ := stringOf(m["type"])
-		switch {
-		case typ == "" || typ == "message":
-			msgs = append(msgs, f.openAIChatMessage(m))
-		case typ == "function_call" || typ == "custom_tool_call":
-			args := m["arguments"]
-			if args == nil {
-				args = m["input"]
-			}
-			call := ToolCallRequestPart{Type: PartTypeToolCall, Name: stringOf(m["name"]), ID: optionalString(m["call_id"]), Arguments: decode(args)}
-			msgs = append(msgs, OutputMessage{Role: RoleAssistant, Parts: Parts{part(call)}})
-		case typ == "function_call_output" || typ == "custom_tool_call_output":
-			result := ToolCallResponsePart{Type: PartTypeToolCallResponse, ID: optionalString(m["call_id"]), Response: toolResponse(m["output"])}
-			msgs = append(msgs, OutputMessage{Role: RoleTool, Parts: Parts{part(result)}})
-		case typ == "reasoning":
-			if p, ok := reasoningItem(m); ok {
-				msgs = append(msgs, OutputMessage{Role: RoleAssistant, Parts: Parts{p}})
-			}
-		case typ == "compaction":
-			msgs = append(msgs, OutputMessage{Role: RoleAssistant, Parts: Parts{part(CompactionPart{Type: PartTypeCompaction, ID: optionalString(m["id"]), Content: optionalString(m["content"])})}})
-		case strings.HasSuffix(typ, "_call"):
-			call := ServerToolCallPart{Type: PartTypeServerToolCall, Name: strings.TrimSuffix(typ, "_call"), ID: optionalString(m["call_id"]), ServerToolCall: GenericServerToolCall(m)}
-			msgs = append(msgs, OutputMessage{Role: RoleAssistant, Parts: Parts{part(call)}})
-		case strings.HasSuffix(typ, "_output") || typ == "mcp_approval_response":
-			result := ServerToolCallResponsePart{Type: PartTypeServerToolCallResponse, ID: optionalString(m["call_id"]), ServerToolCallResponse: GenericServerToolCallResponse(m)}
-			msgs = append(msgs, OutputMessage{Role: RoleTool, Parts: Parts{part(result)}})
-		default:
-			msgs = append(msgs, OutputMessage{Role: RoleAssistant, Parts: Parts{part(GenericPart(m))}})
-		}
-	}
-	return msgs
-}
-
-// reasoningItem joins the summary texts. An item with neither a summary nor encrypted content
-// carries nothing to show:
-// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L69620
-func reasoningItem(m map[string]any) (Part, bool) {
-	var texts []string
-	if summary, ok := m["summary"].([]any); ok {
-		for _, item := range summary {
-			if s, ok := item.(map[string]any); ok && stringOf(s["text"]) != "" {
-				texts = append(texts, stringOf(s["text"]))
-			}
-		}
-	}
-	if len(texts) == 0 && stringOf(m["encrypted_content"]) == "" {
-		return Part{}, false
-	}
-	return part(ReasoningPart{Type: PartTypeReasoning, Content: strings.Join(texts, "\n")}), true
-}

pkg/types/aiobservabilitytypes/message_format_openai_responses_test.go

@@ -1,85 +0,0 @@
-package aiobservabilitytypes
-
-import "testing"
-
-// Cases follow the OpenAI Responses schemas; the first mirrors the raw bodies in the SigNoz/scripts
-// openinference-openai-agents capture, which OpenInference's flattened messages otherwise cover.
-// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L47597
-// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L69691
-func TestFormat_OpenAIResponses(t *testing.T) {
-	runFormatCases(t, []formatCase{
-		{
-			// openinference-openai-agents capture, Response span: instructions outside the history, a reasoning
-			// item without a summary, then the message
-			name:   "InstructionsAndStringInput_ReasoningWithoutSummaryDropped",
-			input:  `{"include": [], "input": [{"content": "A bat and a ball cost $1.10. How much is the ball?", "role": "user"}], "instructions": "You solve problems step by step and give a final answer.", "model": "o3-mini", "tools": []}`,
-			output: `{"id":"resp_1","object":"response","status":"completed","model":"o3-mini-2025-01-31","output":[{"id":"rs_1","summary":[],"type":"reasoning"},{"id":"msg_1","content":[{"annotations":[],"text":"Final Answer: The ball costs $0.05.","type":"output_text","logprobs":[]}],"role":"assistant","status":"completed","type":"message"}],"usage":{"input_tokens":49,"output_tokens":431}}`,
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "You solve problems step by step and give a final answer."}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "A bat and a ball cost $1.10. How much is the ball?"}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "Final Answer: The ball costs $0.05."}], "finish_reason": "stop"}]`,
-		},
-		{
-			// an InputMessage with every InputContent type, then function and custom tool calls with their
-			// outputs, a compaction item and a computer_call_output
-			// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L53827
-			// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L54142
-			name:  "InputItems_EveryContentType_FunctionCallRound",
-			input: `{"model":"gpt-4o-mini","input":[{"role":"user","content":[{"type":"input_text","text":"What is in this image and these files?"},{"type":"input_image","image_url":"https://example.com/duck.jpg","detail":"auto"},{"type":"input_image","file_id":"file-img","detail":"low"},{"type":"input_file","file_id":"file-abc"},{"type":"input_file","filename":"a.pdf","file_data":"data:application/pdf;base64,JVBERi0="},{"type":"input_file","file_url":"https://example.com/a.pdf"}]},{"type":"function_call","call_id":"call_1","name":"get_weather","arguments":"{\"city\":\"Paris\"}"},{"type":"function_call_output","call_id":"call_1","output":"{\"temp_c\":18}"},{"type":"custom_tool_call","call_id":"c1","name":"shell","input":"ls"},{"type":"custom_tool_call_output","call_id":"c1","output":"a.txt"},{"type":"compaction","id":"cmp_1","encrypted_content":"x"},{"type":"computer_call_output","call_id":"cc_1","output":{"type":"computer_screenshot","image_url":"data:image/png;base64,iVBORw0KGgo="}}]}`,
-			wantInput: `[
-			  {"role": "user", "parts": [
-			    {"type": "text", "content": "What is in this image and these files?"},
-			    {"type": "uri", "modality": "image", "uri": "https://example.com/duck.jpg"},
-			    {"type": "file", "modality": "image", "file_id": "file-img"},
-			    {"type": "file", "modality": "document", "file_id": "file-abc", "mime_type": "application/pdf"},
-			    {"type": "blob", "modality": "document", "mime_type": "application/pdf", "content": "JVBERi0="},
-			    {"type": "uri", "modality": "document", "uri": "https://example.com/a.pdf"}
-			  ]},
-			  {"role": "assistant", "parts": [{"type": "tool_call", "name": "get_weather", "id": "call_1", "arguments": {"city": "Paris"}}]},
-			  {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_1", "response": {"temp_c": 18}}]},
-			  {"role": "assistant", "parts": [{"type": "tool_call", "name": "shell", "id": "c1", "arguments": "ls"}]},
-			  {"role": "tool", "parts": [{"type": "tool_call_response", "id": "c1", "response": "a.txt"}]},
-			  {"role": "assistant", "parts": [{"type": "compaction", "id": "cmp_1"}]},
-			  {"role": "tool", "parts": [{"type": "server_tool_call_response", "id": "cc_1", "server_tool_call_response": {"type": "computer_call_output", "call_id": "cc_1", "output": {"type": "computer_screenshot", "image_url": "data:image/png;base64,iVBORw0KGgo="}}}]}
-			]`,
-			wantOutput: `[]`,
-		},
-		{
-			// output items folded into the single choice: a ReasoningItem summary, a WebSearchToolCall, the
-			// OutputMessage with output_text and refusal, then a FunctionToolCall
-			// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L57596
-			name:      "OutputItems_FoldedIntoOneMessage_StatusCompleted",
-			output:    `{"id":"resp_2","object":"response","status":"completed","output":[{"id":"rs_1","type":"reasoning","summary":[{"type":"summary_text","text":"Need the weather first."}],"encrypted_content":"gAAA"},{"id":"ws_1","type":"web_search_call","status":"completed","action":{"type":"search","query":"weather Paris"}},{"id":"msg_1","type":"message","status":"completed","role":"assistant","content":[{"type":"output_text","text":"Checking.","annotations":[]},{"type":"refusal","refusal":"I cannot do that."}]},{"type":"function_call","call_id":"call_1","name":"get_weather","arguments":"{\"city\":\"Paris\"}","id":"fc_1","status":"completed"}]}`,
-			wantInput: `[]`,
-			wantOutput: `[{"role": "assistant", "parts": [
-			  {"type": "reasoning", "content": "Need the weather first."},
-			  {"type": "server_tool_call", "name": "web_search", "server_tool_call": {"id": "ws_1", "type": "web_search_call", "status": "completed", "action": {"type": "search", "query": "weather Paris"}}},
-			  {"type": "text", "content": "Checking."},
-			  {"type": "text", "content": "I cannot do that."},
-			  {"type": "tool_call", "name": "get_weather", "id": "call_1", "arguments": {"city": "Paris"}}
-			], "finish_reason": "stop"}]`,
-		},
-		{
-			// https://github.com/openai/openai-openapi/blob/aeb6a041dc56c011ca9f12127ec898f911100aa0/openapi.yaml#L69763
-			name:       "IncompleteResponse_FinishFromIncompleteDetails",
-			output:     `{"object":"response","status":"incomplete","incomplete_details":{"reason":"max_output_tokens"},"output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Partial"}]}]}`,
-			wantInput:  `[]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "Partial"}], "finish_reason": "length"}]`,
-		},
-		{
-			name:       "StringInput_UserMessage",
-			input:      `{"model":"gpt-4o-mini","input":"Say hi","instructions":"Be brief."}`,
-			wantInput:  `[{"role": "system", "parts": [{"type": "text", "content": "Be brief."}]}, {"role": "user", "parts": [{"type": "text", "content": "Say hi"}]}]`,
-			wantOutput: `[]`,
-		},
-		{
-			// openinference capture, CreateEmbeddings span: an embeddings request also has input and model
-			name:       "EmbeddingsRequest_NotResponses_KeptAsGeneric",
-			input:      `{"input": ["observability is the new monitoring"], "model": "text-embedding-3-small", "encoding_format": "base64"}`,
-			warnings:   []string{"message format not recognised, kept as generic"},
-			wantInput:  `[{"role": "", "parts": [{"type": "generic", "content": "{\"encoding_format\":\"base64\",\"input\":[\"observability is the new monitoring\"],\"model\":\"text-embedding-3-small\"}"}]}]`,
-			wantOutput: `[]`,
-		},
-	})
-}

pkg/types/aiobservabilitytypes/message_format_openinference.go

@@ -1,140 +0,0 @@
-package aiobservabilitytypes
-
-import "github.com/SigNoz/signoz/pkg/valuer"
-
-// Format: the flattened llm.input_messages.N.message.* and llm.output_messages.N.message.* keys, the
-// provider-neutral copy of the conversation the OpenInference instrumentations write on LLM spans. A side
-// without them falls back to the raw provider payload the mapper moved onto gen_ai.*.messages; pipecat
-// streams the reply into output.value alone.
-// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/spec/semantic_conventions.md#L77
-// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/spec/tool_calling.md#L37
-// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/spec/multimodal_attributes.md#L35
-// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/spec/llm_spans.md#L181
-// Handled: message.content, the ordered message.contents items (text, image, audio, video, reasoning),
-// message.tool_calls, the legacy message.function_call_*, tool results, llm.finish_reason. A tool_use
-// content item repeats a call from message.tool_calls and is skipped.
-// Dropped: reasoning id, signature, data and encrypted_content; audio transcripts.
-
-const (
-	openInferenceInputMessages  = "llm.input_messages"
-	openInferenceOutputMessages = "llm.output_messages"
-	openInferenceFinishReason   = "llm.finish_reason"
-)
-
-func (f *formatting) openInference() bool {
-	input := indexed(f.attributes, openInferenceInputMessages)
-	output := indexed(f.attributes, openInferenceOutputMessages)
-	if len(input) == 0 && len(output) == 0 {
-		return false
-	}
-	inputLabel, outputLabel := FormatterOpenInference, FormatterOpenInference
-	if len(input) > 0 {
-		f.out.Input = inputMessages(openInferenceMessages(input))
-	} else if raw := attribute(f.attributes, GenAIInputMessages); raw != nil {
-		var msgs OutputMessages
-		msgs, inputLabel = f.side(decode(raw), RoleUser)
-		f.out.Input = inputMessages(msgs)
-	}
-	if len(output) > 0 {
-		f.out.Output = openInferenceMessages(output)
-		if fr := finishReason(stringOf(attribute(f.attributes, openInferenceFinishReason))); fr != nil {
-			for i := range f.out.Output {
-				f.out.Output[i].FinishReason = fr
-			}
-		}
-	} else if raw := attribute(f.attributes, GenAIOutputMessages); raw != nil {
-		f.out.Output, outputLabel = f.side(decode(raw), RoleAssistant)
-	}
-	f.warnMixedFormats(inputLabel, outputLabel)
-	return true
-}
-
-// openInferenceMessages converts the message buckets and folds runs of assistant messages: the Responses
-// path writes one message per output item, an empty one for a reasoning item without a summary.
-func openInferenceMessages(buckets []map[string]any) OutputMessages {
-	msgs := make(OutputMessages, 0, len(buckets))
-	for _, bucket := range buckets {
-		message, _ := bucket["message"].(map[string]any)
-		msgs = append(msgs, openInferenceMessage(message))
-	}
-	return foldAssistant(msgs)
-}
-
-// openInferenceMessage converts one message.* object. A message without a role is a tool response when
-// it has a tool_call_id, an assistant turn when it has tool calls, else a user turn, as the
-// collector-contrib normalizer decides it:
-// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/fc79b95297776dec6c47490cd6f6026b942bf6fb/processor/genainormalizerprocessor/internal/openinference/messages.go#L473
-func openInferenceMessage(m map[string]any) OutputMessage {
-	msg := OutputMessage{Role: role(stringOf(m["role"])), Parts: Parts{}, Name: optionalString(m["name"])}
-	id := stringOf(m["tool_call_id"])
-	calls := indexed(m, "tool_calls")
-	if msg.Role.IsZero() {
-		switch {
-		case id != "":
-			msg.Role = RoleTool
-		case len(calls) > 0:
-			msg.Role = RoleAssistant
-		default:
-			msg.Role = RoleUser
-		}
-	}
-	if msg.Role == RoleTool && id != "" {
-		msg.Parts = Parts{part(ToolCallResponsePart{Type: PartTypeToolCallResponse, ID: &id, Response: decode(m["content"])})}
-		return msg
-	}
-	if content := stringOf(m["content"]); content != "" {
-		msg.Parts = append(msg.Parts, part(textPart(content)))
-	}
-	msg.Parts = append(msg.Parts, openInferenceContents(m)...)
-	for _, call := range calls {
-		if c, ok := call["tool_call"].(map[string]any); ok {
-			msg.Parts = append(msg.Parts, openInferenceToolCall(c))
-		}
-	}
-	if name := stringOf(m["function_call_name"]); name != "" {
-		msg.Parts = append(msg.Parts, part(ToolCallRequestPart{Type: PartTypeToolCall, Name: name, Arguments: decode(m["function_call_arguments_json"])}))
-	}
-	return msg
-}
-
-// openInferenceContents converts the ordered message.contents items. A text item may come without
-// message_content.type, as the collector-contrib normalizer reads it:
-// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/fc79b95297776dec6c47490cd6f6026b942bf6fb/processor/genainormalizerprocessor/internal/openinference/messages.go#L285
-func openInferenceContents(m map[string]any) Parts {
-	items := indexed(m, "contents")
-	parts := make(Parts, 0, len(items))
-	for _, item := range items {
-		content, _ := item["message_content"].(map[string]any)
-		typ := stringOf(content["type"])
-		if typ == "" && content["text"] != nil {
-			typ = "text"
-		}
-		switch typ {
-		case "text":
-			parts = append(parts, part(textPart(stringOf(content["text"]))))
-		case "reasoning":
-			parts = append(parts, part(ReasoningPart{Type: PartTypeReasoning, Content: stringOf(content["text"])}))
-		case "image", "audio", "video":
-			// the URL sits at message_content.<type>.<type>.url
-			media, _ := content[typ].(map[string]any)
-			inner, _ := media[typ].(map[string]any)
-			if url := stringOf(inner["url"]); url != "" {
-				parts = append(parts, part(mediaPart(url, Modality{valuer.NewString(typ)})))
-			}
-		case "tool_use":
-		default:
-			parts = append(parts, part(GenericPart(content)))
-		}
-	}
-	return parts
-}
-
-// openInferenceToolCall reads a tool_call.* object, {id, function: {name, arguments}}.
-func openInferenceToolCall(call map[string]any) Part {
-	fn, _ := call["function"].(map[string]any)
-	p := ToolCallRequestPart{Type: PartTypeToolCall, Name: stringOf(fn["name"]), ID: optionalString(call["id"])}
-	if args, ok := fn["arguments"]; ok {
-		p.Arguments = decode(args)
-	}
-	return part(p)
-}

pkg/types/aiobservabilitytypes/message_format_openinference_test.go

@@ -1,324 +0,0 @@
-package aiobservabilitytypes
-
-import "testing"
-
-// Captured cases are the llm.input_messages.N.* and llm.output_messages.N.* keys as the collector
-// stored them for the SigNoz/scripts static-telemetry-generator captures.
-func TestFormat_OpenInference(t *testing.T) {
-	runFormatCases(t, []formatCase{
-		{
-			// openinference capture, ChatCompletion span: the flattened keys win over the raw request the
-			// mapper moved onto gen_ai.input.messages
-			name: "FlattenedMessages_RawRequestIgnored",
-			attributes: map[string]any{
-				"llm.input_messages.0.message.role":     "system",
-				"llm.input_messages.0.message.content":  "You are concise.",
-				"llm.input_messages.1.message.role":     "user",
-				"llm.input_messages.1.message.content":  "Say hello in five words.",
-				"llm.output_messages.0.message.role":    "assistant",
-				"llm.output_messages.0.message.content": "Hello! How are you today?",
-				"llm.finish_reason":                     "stop",
-				GenAIInputMessages:                      `{"messages": [{"role": "system", "content": "You are concise."}, {"role": "user", "content": "Say hello in five words."}], "model": "gpt-4o-mini", "max_tokens": 50, "temperature": 0.2}`,
-			},
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "You are concise."}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "Say hello in five words."}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "Hello! How are you today?"}], "finish_reason": "stop"}]`,
-		},
-		{
-			// openinference capture, ChatCompletion span: message.tool_calls and llm.finish_reason
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/spec/tool_calling.md#L37
-			name: "ToolCalls_ToolCallParts_FinishReasonFromLLMKey",
-			attributes: map[string]any{
-				"llm.input_messages.0.message.role":                                       "user",
-				"llm.input_messages.0.message.content":                                    "What is the weather in Bengaluru in celsius?",
-				"llm.output_messages.0.message.role":                                      "assistant",
-				"llm.output_messages.0.message.tool_calls.0.tool_call.id":                 "call_MdYKTRrkshuBG9W5oBXYZaRc",
-				"llm.output_messages.0.message.tool_calls.0.tool_call.function.name":      "get_current_weather",
-				"llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": `{"city":"Bengaluru","unit":"c"}`,
-				"llm.finish_reason": "tool_calls",
-			},
-			wantInput:  `[{"role": "user", "parts": [{"type": "text", "content": "What is the weather in Bengaluru in celsius?"}]}]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "get_current_weather", "id": "call_MdYKTRrkshuBG9W5oBXYZaRc", "arguments": {"city": "Bengaluru", "unit": "c"}}], "finish_reason": "tool_call"}]`,
-		},
-		{
-			// openinference capture, the follow-up ChatCompletion span: tool role messages carry the result
-			// in message.content with their tool_call_id
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/spec/llm_spans.md#L50
-			name: "ToolRoleMessages_ToolCallResponseParts",
-			attributes: map[string]any{
-				"llm.input_messages.0.message.role":                                      "user",
-				"llm.input_messages.0.message.content":                                   "What's the weather in Paris and in London? Use the tool for each.",
-				"llm.input_messages.1.message.role":                                      "assistant",
-				"llm.input_messages.1.message.tool_calls.0.tool_call.id":                 "call_SW4hst7nzysgvWSAz4OdyK9D",
-				"llm.input_messages.1.message.tool_calls.0.tool_call.function.name":      "get_current_weather",
-				"llm.input_messages.1.message.tool_calls.0.tool_call.function.arguments": `{"city": "Paris"}`,
-				"llm.input_messages.1.message.tool_calls.1.tool_call.id":                 "call_si8wiETYGpooMB7gMXSgMQpk",
-				"llm.input_messages.1.message.tool_calls.1.tool_call.function.name":      "get_current_weather",
-				"llm.input_messages.1.message.tool_calls.1.tool_call.function.arguments": `{"city": "London"}`,
-				"llm.input_messages.2.message.role":                                      "tool",
-				"llm.input_messages.2.message.tool_call_id":                              "call_SW4hst7nzysgvWSAz4OdyK9D",
-				"llm.input_messages.2.message.content":                                   `{"city": "Paris", "temp_c": 18, "summary": "Clear"}`,
-				"llm.input_messages.3.message.role":                                      "tool",
-				"llm.input_messages.3.message.tool_call_id":                              "call_si8wiETYGpooMB7gMXSgMQpk",
-				"llm.input_messages.3.message.content":                                   `{"city": "London", "temp_c": 18, "summary": "Clear"}`,
-				"llm.output_messages.0.message.role":                                     "assistant",
-				"llm.output_messages.0.message.content":                                  "The current weather is as follows:\n\n- **Paris**: 18°C, Clear\n- **London**: 18°C, Clear",
-				"llm.finish_reason":                                                      "stop",
-			},
-			wantInput: `[
-			  {"role": "user", "parts": [{"type": "text", "content": "What's the weather in Paris and in London? Use the tool for each."}]},
-			  {"role": "assistant", "parts": [
-			    {"type": "tool_call", "name": "get_current_weather", "id": "call_SW4hst7nzysgvWSAz4OdyK9D", "arguments": {"city": "Paris"}},
-			    {"type": "tool_call", "name": "get_current_weather", "id": "call_si8wiETYGpooMB7gMXSgMQpk", "arguments": {"city": "London"}}
-			  ]},
-			  {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_SW4hst7nzysgvWSAz4OdyK9D", "response": {"city": "Paris", "temp_c": 18, "summary": "Clear"}}]},
-			  {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_si8wiETYGpooMB7gMXSgMQpk", "response": {"city": "London", "temp_c": 18, "summary": "Clear"}}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "The current weather is as follows:\n\n- **Paris**: 18°C, Clear\n- **London**: 18°C, Clear"}], "finish_reason": "stop"}]`,
-		},
-		{
-			// openinference capture, vision call; base64 trimmed. An image as a data URL:
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/spec/multimodal_attributes.md#L47
-			name: "ContentsTextAndImage_TextAndBlobParts",
-			attributes: map[string]any{
-				"llm.input_messages.0.message.role":                                       "user",
-				"llm.input_messages.0.message.contents.0.message_content.type":            "text",
-				"llm.input_messages.0.message.contents.0.message_content.text":            "What animal is in this image? One sentence.",
-				"llm.input_messages.0.message.contents.1.message_content.type":            "image",
-				"llm.input_messages.0.message.contents.1.message_content.image.image.url": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBD",
-				"llm.output_messages.0.message.role":                                      "assistant",
-				"llm.output_messages.0.message.content":                                   "The animal in the image is a duck.",
-				"llm.finish_reason":                                                       "stop",
-			},
-			wantInput: `[{"role": "user", "parts": [
-			  {"type": "text", "content": "What animal is in this image? One sentence."},
-			  {"type": "blob", "modality": "image", "mime_type": "image/jpeg", "content": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBD"}
-			]}]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "The animal in the image is a duck."}], "finish_reason": "stop"}]`,
-		},
-		{
-			// openinference-openai-agents capture, Response span: one output message per item, an empty one
-			// for a reasoning item without a summary
-			name: "ResponsesOutputItems_AssistantMessagesFolded_EmptyDropped",
-			attributes: map[string]any{
-				"llm.input_messages.0.message.role":                                  "system",
-				"llm.input_messages.0.message.content":                               "You solve problems step by step and give a final answer.",
-				"llm.input_messages.1.message.role":                                  "user",
-				"llm.input_messages.1.message.content":                               "A bat and a ball cost $1.10. How much is the ball?",
-				"llm.output_messages.0.message.role":                                 "assistant",
-				"llm.output_messages.1.message.role":                                 "assistant",
-				"llm.output_messages.1.message.contents.0.message_content.type":      "text",
-				"llm.output_messages.1.message.contents.0.message_content.text":      "Final Answer: The ball costs $0.05.",
-				"llm.output_messages.2.message.role":                                 "assistant",
-				"llm.output_messages.2.message.tool_calls.0.tool_call.id":            "call_1",
-				"llm.output_messages.2.message.tool_calls.0.tool_call.function.name": "done",
-			},
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "You solve problems step by step and give a final answer."}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "A bat and a ball cost $1.10. How much is the ball?"}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [
-			  {"type": "text", "content": "Final Answer: The ball costs $0.05."},
-			  {"type": "tool_call", "name": "done", "id": "call_1"}
-			]}]`,
-		},
-		{
-			// openinference capture, the ChatCompletion span of a failed call
-			name: "InputOnly_NoOutput",
-			attributes: map[string]any{
-				"llm.input_messages.0.message.role":    "user",
-				"llm.input_messages.0.message.content": "this should fail",
-			},
-			wantInput:  `[{"role": "user", "parts": [{"type": "text", "content": "this should fail"}]}]`,
-			wantOutput: `[]`,
-		},
-		{
-			// the strands processor keeps the SDK's gen_ai.system_instructions on the span and inserts the same
-			// prompt as message 0
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/python/instrumentation/openinference-instrumentation-strands-agents/src/openinference/instrumentation/strands_agents/processor.py#L152
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/python/instrumentation/openinference-instrumentation-strands-agents/src/openinference/instrumentation/strands_agents/processor.py#L293
-			name: "StrandsSystemInstructionsRepeated_OneSystemMessage",
-			attributes: map[string]any{
-				"llm.input_messages.0.message.role":     "system",
-				"llm.input_messages.0.message.content":  "You are concise.",
-				"llm.input_messages.1.message.role":     "user",
-				"llm.input_messages.1.message.content":  "hi",
-				"llm.output_messages.0.message.role":    "assistant",
-				"llm.output_messages.0.message.content": "Hello.",
-				GenAISystemInstructions:                 `[{"type": "text", "content": "You are concise."}]`,
-			},
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "You are concise."}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "hi"}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "Hello."}]}]`,
-		},
-		{
-			// pipecat writes the request as llm.input_messages and the streamed reply as output.value alone,
-			// which the mapper moves onto gen_ai.output.messages
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/python/instrumentation/openinference-instrumentation-pipecat/src/openinference/instrumentation/pipecat/_attributes.py#L430
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/python/instrumentation/openinference-instrumentation-pipecat/src/openinference/instrumentation/pipecat/_observer.py#L628
-			name: "OutputKeysAbsent_RawOutputFormatted_Warns",
-			attributes: map[string]any{
-				"llm.input_messages.0.message.role":    "user",
-				"llm.input_messages.0.message.content": "hi",
-				GenAIOutputMessages:                    "Hello there.",
-			},
-			warnings:   []string{"bare assistant text, role assumed", "output formatted as text, input as openinference"},
-			wantInput:  `[{"role": "user", "parts": [{"type": "text", "content": "hi"}]}]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "Hello there."}]}]`,
-		},
-		{
-			// message.role left unset by some instrumentors: tool_call_id, tool_calls or neither decide it,
-			// as the collector-contrib normalizer does
-			// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/fc79b95297776dec6c47490cd6f6026b942bf6fb/processor/genainormalizerprocessor/internal/openinference/messages.go#L473
-			name: "MissingRole_InferredFromToolFields",
-			attributes: map[string]any{
-				"llm.input_messages.0.message.content":                                   "Weather in Oslo?",
-				"llm.input_messages.1.message.tool_calls.0.tool_call.id":                 "call_9",
-				"llm.input_messages.1.message.tool_calls.0.tool_call.function.name":      "get_weather",
-				"llm.input_messages.1.message.tool_calls.0.tool_call.function.arguments": `{"city":"Oslo"}`,
-				"llm.input_messages.2.message.tool_call_id":                              "call_9",
-				"llm.input_messages.2.message.content":                                   "4C, cloudy",
-			},
-			wantInput: `[
-			  {"role": "user", "parts": [{"type": "text", "content": "Weather in Oslo?"}]},
-			  {"role": "assistant", "parts": [{"type": "tool_call", "name": "get_weather", "id": "call_9", "arguments": {"city": "Oslo"}}]},
-			  {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_9", "response": "4C, cloudy"}]}
-			]`,
-			wantOutput: `[]`,
-		},
-		{
-			// message_content.type left unset on text items by some instrumentors, as the collector-contrib
-			// normalizer reads them
-			// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/fc79b95297776dec6c47490cd6f6026b942bf6fb/processor/genainormalizerprocessor/internal/openinference/messages.go#L285
-			name: "UntypedContentWithText_TextPart",
-			attributes: map[string]any{
-				"llm.input_messages.0.message.role":                            "user",
-				"llm.input_messages.0.message.contents.0.message_content.text": "Summarise this.",
-				"llm.input_messages.0.message.contents.1.message_content.text": "Second block.",
-			},
-			wantInput:  `[{"role": "user", "parts": [{"type": "text", "content": "Summarise this."}, {"type": "text", "content": "Second block."}]}]`,
-			wantOutput: `[]`,
-		},
-		{
-			// an image given by URL
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/spec/multimodal_attributes.md#L42
-			name: "ImageURL_UriPart",
-			attributes: map[string]any{
-				"llm.input_messages.0.message.role":                                       "user",
-				"llm.input_messages.0.message.contents.0.message_content.type":            "image",
-				"llm.input_messages.0.message.contents.0.message_content.image.image.url": "https://example.com/duck.jpg",
-			},
-			wantInput:  `[{"role": "user", "parts": [{"type": "uri", "modality": "image", "uri": "https://example.com/duck.jpg"}]}]`,
-			wantOutput: `[]`,
-		},
-		{
-			// audio by URL with its transcript, or as the data URI built from a Chat Completions input_audio
-			// block; the transcript has no part to go to
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/spec/multimodal_attributes.md#L55
-			name: "Audio_UriAndBlobParts_TranscriptDropped",
-			attributes: map[string]any{
-				"llm.input_messages.0.message.role":                                              "user",
-				"llm.input_messages.0.message.contents.0.message_content.type":                   "audio",
-				"llm.input_messages.0.message.contents.0.message_content.audio.audio.url":        "https://example.com/hello.mp3",
-				"llm.input_messages.0.message.contents.0.message_content.audio.audio.transcript": "Hello, how are you?",
-				"llm.input_messages.0.message.contents.1.message_content.type":                   "audio",
-				"llm.input_messages.0.message.contents.1.message_content.audio.audio.url":        "data:audio/wav;base64,UklGRg==",
-			},
-			wantInput: `[{"role": "user", "parts": [
-			  {"type": "uri", "modality": "audio", "uri": "https://example.com/hello.mp3"},
-			  {"type": "blob", "modality": "audio", "mime_type": "audio/wav", "content": "UklGRg=="}
-			]}]`,
-			wantOutput: `[]`,
-		},
-		{
-			// reasoning items; id, signature, data and encrypted_content have no field on the reasoning part
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/spec/llm_spans.md#L181
-			name: "Reasoning_ReasoningParts_ProviderKeysDropped",
-			attributes: map[string]any{
-				"llm.output_messages.0.message.role":                                         "assistant",
-				"llm.output_messages.0.message.contents.0.message_content.type":              "reasoning",
-				"llm.output_messages.0.message.contents.0.message_content.text":              "The ball must cost less than a dollar.",
-				"llm.output_messages.0.message.contents.0.message_content.signature":         "EuYBCkQYAiJA",
-				"llm.output_messages.0.message.contents.1.message_content.type":              "reasoning",
-				"llm.output_messages.0.message.contents.1.message_content.data":              "EmwKAhgBEgy3",
-				"llm.output_messages.0.message.contents.2.message_content.type":              "reasoning",
-				"llm.output_messages.0.message.contents.2.message_content.id":                "rs_abc123",
-				"llm.output_messages.0.message.contents.2.message_content.encrypted_content": "gAAAAA==",
-				"llm.output_messages.0.message.contents.3.message_content.type":              "text",
-				"llm.output_messages.0.message.contents.3.message_content.text":              "The ball costs 5 cents.",
-			},
-			wantInput: `[]`,
-			wantOutput: `[{"role": "assistant", "parts": [
-			  {"type": "reasoning", "content": "The ball must cost less than a dollar."},
-			  {"type": "reasoning", "content": ""},
-			  {"type": "reasoning", "content": ""},
-			  {"type": "text", "content": "The ball costs 5 cents."}
-			]}]`,
-		},
-		{
-			// a tool call repeated as an ordered tool_use content item; message.tool_calls stays the one
-			// source. llm.finish_reason is Anthropic's stop_reason as the instrumentation passes it through
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/spec/tool_calling.md#L63
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/python/instrumentation/openinference-instrumentation-anthropic/src/openinference/instrumentation/anthropic/_wrappers.py#L938
-			name: "ToolUseBesideToolCalls_OneToolCallPart",
-			attributes: map[string]any{
-				"llm.output_messages.0.message.role":                                      "assistant",
-				"llm.output_messages.0.message.contents.0.message_content.type":           "text",
-				"llm.output_messages.0.message.contents.0.message_content.text":           "Let me check.",
-				"llm.output_messages.0.message.contents.1.message_content.type":           "tool_use",
-				"llm.output_messages.0.message.contents.1.tool_call.id":                   "toolu_01",
-				"llm.output_messages.0.message.contents.1.tool_call.function.name":        "get_weather",
-				"llm.output_messages.0.message.contents.1.tool_call.function.arguments":   `{"city": "Paris"}`,
-				"llm.output_messages.0.message.tool_calls.0.tool_call.id":                 "toolu_01",
-				"llm.output_messages.0.message.tool_calls.0.tool_call.function.name":      "get_weather",
-				"llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": `{"city": "Paris"}`,
-				"llm.finish_reason": "tool_use",
-			},
-			wantInput: `[]`,
-			wantOutput: `[{"role": "assistant", "parts": [
-			  {"type": "text", "content": "Let me check."},
-			  {"type": "tool_call", "name": "get_weather", "id": "toolu_01", "arguments": {"city": "Paris"}}
-			], "finish_reason": "tool_call"}]`,
-		},
-		{
-			// a content type the spec does not name is kept as sent
-			name: "UnknownContentType_GenericPart",
-			attributes: map[string]any{
-				"llm.input_messages.0.message.role":                                            "user",
-				"llm.input_messages.0.message.contents.0.message_content.type":                 "document",
-				"llm.input_messages.0.message.contents.0.message_content.document.document.id": "file_1",
-			},
-			wantInput:  `[{"role": "user", "parts": [{"type": "document", "document": {"document": {"id": "file_1"}}}]}]`,
-			wantOutput: `[]`,
-		},
-		{
-			// the legacy function_call keys
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/spec/semantic_conventions.md#L111
-			name: "LegacyFunctionCall_ToolCallPart",
-			attributes: map[string]any{
-				"llm.output_messages.0.message.role":                         "assistant",
-				"llm.output_messages.0.message.function_call_name":           "multiply",
-				"llm.output_messages.0.message.function_call_arguments_json": `{"x": 2}`,
-			},
-			wantInput:  `[]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "multiply", "arguments": {"x": 2}}]}]`,
-		},
-		{
-			// a tool result with message.name naming the function
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/spec/llm_spans.md#L50
-			name: "ToolResultWithName_NameKept",
-			attributes: map[string]any{
-				"llm.input_messages.0.message.role":         "tool",
-				"llm.input_messages.0.message.name":         "multiply",
-				"llm.input_messages.0.message.tool_call_id": "call_62136355",
-				"llm.input_messages.0.message.content":      "2001",
-			},
-			wantInput:  `[{"role": "tool", "name": "multiply", "parts": [{"type": "tool_call_response", "id": "call_62136355", "response": "2001"}]}]`,
-			wantOutput: `[]`,
-		},
-	})
-}

pkg/types/aiobservabilitytypes/message_format_openrouter.go

@@ -1,47 +0,0 @@
-package aiobservabilitytypes
-
-import "strings"
-
-// Format: the {completion, reasoning, tools, rawRequest} answer object OpenRouter's broadcast writes as
-// gen_ai.completion on its LLM Generation spans, beside the OpenAI chat {messages} request it writes as
-// gen_ai.prompt; the span mapper moves both onto gen_ai.*.messages. A tool call arrives as the tool name
-// followed by its JSON arguments in completion, so it is split against the names in tools.
-// https://openrouter.ai/docs/guides/features/broadcast/otel-collector
-
-func isOpenRouterCompletion(m map[string]any) bool {
-	_, ok := m["completion"].(string)
-	return ok
-}
-
-func (f *formatting) openRouterCompletion(m map[string]any) OutputMessages {
-	msg := OutputMessage{Role: RoleAssistant, Parts: Parts{}}
-	if reasoning := stringOf(m["reasoning"]); reasoning != "" {
-		msg.Parts = append(msg.Parts, part(ReasoningPart{Type: PartTypeReasoning, Content: reasoning}))
-	}
-	completion := stringOf(m["completion"])
-	if call, ok := openRouterToolCall(completion, m["tools"]); ok {
-		msg.Parts = append(msg.Parts, call)
-		msg.FinishReason = finishReason("tool_calls")
-	} else if completion != "" {
-		msg.Parts = append(msg.Parts, part(textPart(completion)))
-	}
-	return OutputMessages{msg}
-}
-
-func openRouterToolCall(completion string, tools any) (Part, bool) {
-	list, _ := tools.([]any)
-	for _, item := range list {
-		tool, _ := item.(map[string]any)
-		fn, _ := tool["function"].(map[string]any)
-		name := stringOf(fn["name"])
-		if name == "" || !strings.HasPrefix(completion, name) {
-			continue
-		}
-		args, ok := decode(strings.TrimPrefix(completion, name)).(map[string]any)
-		if !ok {
-			continue
-		}
-		return part(ToolCallRequestPart{Type: PartTypeToolCall, Name: name, Arguments: args}), true
-	}
-	return Part{}, false
-}

pkg/types/aiobservabilitytypes/message_format_openrouter_test.go

@@ -1,35 +0,0 @@
-package aiobservabilitytypes
-
-import "testing"
-
-// Captured cases are gen_ai.input.messages and gen_ai.output.messages as the collector stored them for
-// the SigNoz/scripts openrouter-gateway capture, LLM Generation spans.
-func TestFormat_OpenRouter(t *testing.T) {
-	runFormatCases(t, []formatCase{
-		{
-			// the completion is the tool name followed by the JSON arguments
-			name:       "CompletionIsToolNameAndArguments_ToolCallPart",
-			input:      `{"messages":[{"role":"user","content":"What's the weather in Bengaluru? Use the tool."}]}`,
-			output:     `{"completion":"get_current_weather{\"city\":\"Bengaluru\"}","reasoning":null,"tools":[{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather for a city.","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}],"rawRequest":{"model":"openai/gpt-4o-mini","max_tokens":80,"tool_choice":"auto"}}`,
-			wantInput:  `[{"role": "user", "parts": [{"type": "text", "content": "What's the weather in Bengaluru? Use the tool."}]}]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "get_current_weather", "arguments": {"city": "Bengaluru"}}], "finish_reason": "tool_call"}]`,
-		},
-		{
-			name:   "CompletionWithReasoning_ReasoningThenText",
-			input:  `{"messages":[{"role":"system","content":"You are concise."},{"role":"user","content":"Say hello in five words."}]}`,
-			output: `{"completion":"Hello there, how are you?","reasoning":"A greeting of exactly five words.","tools":[],"rawRequest":{"model":"openai/gpt-4o-mini"}}`,
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "You are concise."}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "Say hello in five words."}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "reasoning", "content": "A greeting of exactly five words."}, {"type": "text", "content": "Hello there, how are you?"}]}]`,
-		},
-		{
-			// a completion that only starts with a tool name stays text
-			name:       "CompletionStartsWithToolNameButNoArguments_Text",
-			output:     `{"completion":"get_current_weather is the tool to call.","reasoning":null,"tools":[{"type":"function","function":{"name":"get_current_weather"}}]}`,
-			wantInput:  `[]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "get_current_weather is the tool to call."}]}]`,
-		},
-	})
-}

pkg/types/aiobservabilitytypes/message_format_semconv.go

@@ -6,20 +6,15 @@ import (
 )
 
 // Format: the OTel shape itself, [{role, parts}].
-// Written by: litellm, langchain, openllmetry, portkey, traceloop LLM spans, pydantic-ai.
-// Decoding is the genai types. Repairs: "text" used for "content" on text parts; OpenAI content blocks
-// left inside parts; tool call arguments as a JSON string and "result" for "response", as pydantic-ai
-// writes them:
-// https://github.com/pydantic/pydantic-ai/blob/9ac853bb84ffed7ba0edccb026c5b4f3228c1de7/pydantic_ai_slim/pydantic_ai/messages.py#L3130
-// https://github.com/pydantic/pydantic-ai/blob/9ac853bb84ffed7ba0edccb026c5b4f3228c1de7/pydantic_ai_slim/pydantic_ai/messages.py#L1724
-// The choice finish_reason as the provider spells it, as litellm copies it:
-// https://github.com/BerriAI/litellm/blob/9d72bae336ec0d14c41651adcb0d2e1149721f46/litellm/integrations/opentelemetry.py#L2822
-// Not yet: "thinking" for "reasoning".
+// Written by: litellm, langchain, openllmetry, portkey, traceloop LLM spans.
+// Decoding is the genai types. Repairs: "text" used for "content" on text parts; OpenAI content
+// blocks left inside parts.
+// Not yet: "thinking" for "reasoning"; "result" or "output" for "response" on tool call responses.
 
 // semconv decodes messages that already follow the schema through the genai types. A part of a
 // type the schema does not name is a provider content block, which contentPart converts.
 func (f *formatting) semconv(list []any) OutputMessages {
-	data, err := json.Marshal(semconvRepairs(list))
+	data, err := json.Marshal(semconvTextKey(list))
 	if err != nil {
 		f.warn("messages could not be encoded: %s", err)
 		return OutputMessages{genericMessage(list)}
@@ -33,9 +28,6 @@ func (f *formatting) semconv(list []any) OutputMessages {
 		if msgs[i].Parts == nil {
 			msgs[i].Parts = Parts{}
 		}
-		if msgs[i].FinishReason != nil {
-			msgs[i].FinishReason = finishReason(msgs[i].FinishReason.StringValue())
-		}
 		for j, p := range msgs[i].Parts {
 			if block, ok := p.Value.(GenericPart); ok {
 				msgs[i].Parts[j] = contentPart(map[string]any(block))
@@ -45,9 +37,9 @@ func (f *formatting) semconv(list []any) OutputMessages {
 	return msgs
 }
 
-// semconvRepairs copies the parts whose keys SDKs misspell into the schema's spelling. The input
-// is left untouched, it is also the span's raw attribute.
-func semconvRepairs(list []any) []any {
+// semconvTextKey copies the text and reasoning parts some SDKs write with "text" in place of the
+// schema's "content". The input is left untouched, it is also the span's raw attribute.
+func semconvTextKey(list []any) []any {
 	out := make([]any, 0, len(list))
 	for _, item := range list {
 		m, ok := item.(map[string]any)
@@ -58,8 +50,13 @@ func semconvRepairs(list []any) []any {
 		}
 		fixedParts := make([]any, 0, len(parts))
 		for _, p := range parts {
-			if part, ok := p.(map[string]any); ok {
-				p = semconvRepairPart(part)
+			part, ok := p.(map[string]any)
+			typ := stringOf(part["type"])
+			if ok && (typ == "text" || typ == "reasoning") && part["content"] == nil && part["text"] != nil {
+				fixed := maps.Clone(part)
+				fixed["content"] = fixed["text"]
+				delete(fixed, "text")
+				p = fixed
 			}
 			fixedParts = append(fixedParts, p)
 		}
@@ -69,24 +66,3 @@ func semconvRepairs(list []any) []any {
 	}
 	return out
 }
-
-func semconvRepairPart(part map[string]any) map[string]any {
-	fixed := maps.Clone(part)
-	switch stringOf(part["type"]) {
-	case "text", "reasoning":
-		if part["content"] == nil && part["text"] != nil {
-			fixed["content"] = fixed["text"]
-			delete(fixed, "text")
-		}
-	case "tool_call":
-		if args, ok := part["arguments"].(string); ok {
-			fixed["arguments"] = decode(args)
-		}
-	case "tool_call_response":
-		if _, ok := part["response"]; !ok && part["result"] != nil {
-			fixed["response"] = part["result"]
-			delete(fixed, "result")
-		}
-	}
-	return fixed
-}

pkg/types/aiobservabilitytypes/message_format_semconv_test.go

@@ -157,34 +157,3 @@ func TestFormat_Semconv(t *testing.T) {
 		},
 	})
 }
-
-func TestFormat_Semconv_Repairs(t *testing.T) {
-	runFormatCases(t, []formatCase{
-		{
-			// pydantic-ai keeps string tool call arguments as sent, writes the tool result under result, and the
-			// instructions as a gen_ai.system_instructions text part:
-			// https://github.com/pydantic/pydantic-ai/blob/9ac853bb84ffed7ba0edccb026c5b4f3228c1de7/pydantic_ai_slim/pydantic_ai/messages.py#L3130
-			// https://github.com/pydantic/pydantic-ai/blob/9ac853bb84ffed7ba0edccb026c5b4f3228c1de7/pydantic_ai_slim/pydantic_ai/messages.py#L1724
-			// https://github.com/pydantic/pydantic-ai/blob/9ac853bb84ffed7ba0edccb026c5b4f3228c1de7/pydantic_ai_slim/pydantic_ai/models/instrumented.py#L292
-			name:       "PydanticAI_ArgumentsDecoded_ResultBecomesResponse_InstructionsLead",
-			input:      `[{"role":"user","parts":[{"type":"text","content":"Weather in Paris?"}]},{"role":"assistant","parts":[{"type":"tool_call","id":"call_stub1","name":"get_weather","arguments":"{\"city\": \"Paris\"}"}],"finish_reason":"tool_call"},{"role":"user","parts":[{"type":"tool_call_response","id":"call_stub1","name":"get_weather","result":{"city":"Paris","temp_c":18}}]}]`,
-			output:     `[{"role":"assistant","parts":[{"type":"text","content":"Paris is clear at 18C."}],"finish_reason":"stop"}]`,
-			attributes: map[string]any{GenAISystemInstructions: `[{"type":"text","content":"Use the tool."}]`},
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "Use the tool."}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "Weather in Paris?"}]},
-			  {"role": "assistant", "parts": [{"type": "tool_call", "name": "get_weather", "id": "call_stub1", "arguments": {"city": "Paris"}}]},
-			  {"role": "user", "parts": [{"type": "tool_call_response", "id": "call_stub1", "response": {"city": "Paris", "temp_c": 18}}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "Paris is clear at 18C."}], "finish_reason": "stop"}]`,
-		},
-		{
-			// litellm copies the choice finish_reason into the semconv message as OpenAI spells it:
-			// https://github.com/BerriAI/litellm/blob/9d72bae336ec0d14c41651adcb0d2e1149721f46/litellm/integrations/opentelemetry.py#L2822
-			name:       "LiteLLMFinishReasonSpelling_Normalised",
-			output:     `[{"role": "assistant", "parts": [], "finish_reason": "tool_calls"}]`,
-			wantInput:  `[]`,
-			wantOutput: `[{"role": "assistant", "parts": [], "finish_reason": "tool_call"}]`,
-		},
-	})
-}

pkg/types/aiobservabilitytypes/message_format_test.go

@@ -3,6 +3,7 @@ package aiobservabilitytypes
 import (
 	"encoding/json"
 	"maps"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -25,7 +26,7 @@ func runFormatCases(t *testing.T, testCases []formatCase) {
 		t.Run(testCase.name, func(t *testing.T) {
 			attributes := map[string]any{GenAIInputMessages: testCase.input, GenAIOutputMessages: testCase.output}
 			maps.Copy(attributes, testCase.attributes)
-			formatted := FormatMessages(attributes)
+			formatted := FormatMessages(document(attributes))
 			assert.Equal(t, append([]string{}, testCase.warnings...), formatted.Warnings)
 
 			input, err := json.Marshal(formatted.Input)
@@ -38,6 +39,25 @@ func runFormatCases(t *testing.T, testCases []formatCase) {
 	}
 }
 
+// document nests flat dotted keys the way the JSON column stores them.
+func document(flat map[string]any) map[string]any {
+	doc := map[string]any{}
+	for key, value := range flat {
+		segments := strings.Split(key, ".")
+		current := doc
+		for _, segment := range segments[:len(segments)-1] {
+			child, ok := current[segment].(map[string]any)
+			if !ok {
+				child = map[string]any{}
+				current[segment] = child
+			}
+			current = child
+		}
+		current[segments[len(segments)-1]] = value
+	}
+	return doc
+}
+
 func TestFormat(t *testing.T) {
 	runFormatCases(t, []formatCase{
 		{
@@ -92,16 +112,6 @@ So, the ball costs 5 cents.`,
 			wantInput:  `[{"role": "", "parts": [{"type": "generic", "content": "{\"city\":\"Paris\",\"temp_c\":21}"}]}]`,
 			wantOutput: `[]`,
 		},
-		{
-			// openinference capture, CreateEmbeddings span: the mapper moved the request and the vectors onto
-			// the message attributes and copied the span kind onto gen_ai.operation.name
-			name:       "EmbeddingsOperation_EmptyLists",
-			input:      `{"input": ["observability is the new monitoring"], "model": "text-embedding-3-small"}`,
-			output:     `{"data":[{"embedding":[-0.0039,0.0203],"index":0}],"model":"text-embedding-3-small"}`,
-			attributes: map[string]any{GenAIOperationName: "EMBEDDING", GenAIRequestModel: "text-embedding-3-small"},
-			wantInput:  `[]`,
-			wantOutput: `[]`,
-		},
 		{
 			name:       "NoMessageAttributes_EmptyLists",
 			attributes: map[string]any{GenAIRequestModel: "gpt-4o", GenAIToolName: "get_weather"},
@@ -110,113 +120,3 @@ So, the ball costs 5 cents.`,
 		},
 	})
 }
-
-func TestIndexed(t *testing.T) {
-	testCases := []struct {
-		name string
-		doc  map[string]any
-		want []map[string]any
-	}{
-		{
-			name: "NumericKeys_SortedNumerically",
-			doc:  map[string]any{"llm.input_messages.10.message.role": "user", "llm.input_messages.2.message.role": "system", "llm.input_messages.x.message.role": "skipped"},
-			want: []map[string]any{{"message": map[string]any{"role": "system"}}, {"message": map[string]any{"role": "user"}}},
-		},
-		{
-			name: "ScalarAtKey_EntriesBesideItRead",
-			doc:  map[string]any{"llm.input_messages": "raw", "llm.input_messages.0.message.role": "user"},
-			want: []map[string]any{{"message": map[string]any{"role": "user"}}},
-		},
-		{
-			name: "ScalarAndObjectAtSamePath_ObjectKept",
-			doc:  map[string]any{"llm.input_messages.0.message": "raw", "llm.input_messages.0.message.role": "user"},
-			want: []map[string]any{{"message": map[string]any{"role": "user"}}},
-		},
-		{name: "Absent_Empty", doc: map[string]any{}, want: []map[string]any{}},
-	}
-	for _, testCase := range testCases {
-		t.Run(testCase.name, func(t *testing.T) {
-			assert.Equal(t, testCase.want, indexed(testCase.doc, openInferenceInputMessages))
-		})
-	}
-}
-
-func TestFormat_SystemInstructions(t *testing.T) {
-	input := `[{"role":"user","parts":[{"type":"text","content":"hi"}]}]`
-	runFormatCases(t, []formatCase{
-		{
-			// the langchain instrumentation moves system messages out of the history into gen_ai.system_instructions,
-			// a JSON list of parts:
-			// https://github.com/traceloop/openllmetry/blob/93429cfb9767e6b51db6cb35f6b70f758117669a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/span_utils.py#L304
-			name:       "ListOfParts_LeadsInput",
-			input:      input,
-			attributes: map[string]any{GenAISystemInstructions: `[{"type": "text", "content": "You are concise."}]`},
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "You are concise."}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "hi"}]}
-			]`,
-			wantOutput: `[]`,
-		},
-		{
-			// the OpenInference strands processor keeps gen_ai.system_instructions on the span and inserts the
-			// same prompt as message 0
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/python/instrumentation/openinference-instrumentation-strands-agents/src/openinference/instrumentation/strands_agents/processor.py#L152
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/python/instrumentation/openinference-instrumentation-strands-agents/src/openinference/instrumentation/strands_agents/processor.py#L293
-			name:       "SameTextLeadsInput_Skipped",
-			input:      `[{"role":"system","parts":[{"type":"text","content":"You are concise."}]},{"role":"user","parts":[{"type":"text","content":"hi"}]}]`,
-			attributes: map[string]any{GenAISystemInstructions: `[{"type": "text", "content": "You are concise."}]`},
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "You are concise."}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "hi"}]}
-			]`,
-			wantOutput: `[]`,
-		},
-		{
-			name:       "BareString_TextPart",
-			input:      input,
-			attributes: map[string]any{GenAISystemInstructions: "be brief"},
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "be brief"}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "hi"}]}
-			]`,
-			wantOutput: `[]`,
-		},
-		{
-			// a create_agent span carries instructions alone
-			name:       "NoMessages_Nothing",
-			attributes: map[string]any{GenAISystemInstructions: "be brief"},
-			wantInput:  `[]`,
-			wantOutput: `[]`,
-		},
-	})
-}
-
-func TestFormat_FinishReasons(t *testing.T) {
-	output := `[{"role":"assistant","parts":[{"type":"text","content":"a"}],"finish_reason":"stop"},{"role":"assistant","parts":[{"type":"text","content":"b"}]}]`
-	runFormatCases(t, []formatCase{
-		{
-			name:       "List_OverridesMessages",
-			output:     output,
-			attributes: map[string]any{GenAIResponseFinishReasons: []any{"tool_calls", "length"}},
-			wantInput:  `[]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "a"}], "finish_reason": "tool_call"}, {"role": "assistant", "parts": [{"type": "text", "content": "b"}], "finish_reason": "length"}]`,
-		},
-		{
-			name:       "LengthMismatch_LeavesMessages_Warns",
-			output:     output,
-			attributes: map[string]any{GenAIResponseFinishReasons: []any{"stop"}},
-			warnings:   []string{"gen_ai.response.finish_reasons has 1 entries for 2 output messages"},
-			wantInput:  `[]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "a"}], "finish_reason": "stop"}, {"role": "assistant", "parts": [{"type": "text", "content": "b"}]}]`,
-		},
-		{
-			// llm.finish_reason copied by the mapper as a bare string, in the Vercel AI SDK spelling:
-			// https://github.com/vercel/ai/blob/a5e92fe3c692f8e152fe1eb9ef50a4e734ad0ffb/packages/provider/src/language-model/v2/language-model-v2-finish-reason.ts
-			name:       "BareString_SingleChoice",
-			output:     `{"choices":[{"finish_reason":"stop","message":{"role":"assistant","content":"hi"}}]}`,
-			attributes: map[string]any{GenAIResponseFinishReasons: "tool-calls"},
-			wantInput:  `[]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "hi"}], "finish_reason": "tool_call"}]`,
-		},
-	})
-}

pkg/types/aiobservabilitytypes/message_format_tool.go

@@ -1,15 +1,12 @@
 package aiobservabilitytypes
 
 // Format: gen_ai.tool.call.arguments and gen_ai.tool.call.result on an execute_tool span, each an
-// object or its JSON string:
-// https://github.com/open-telemetry/semantic-conventions-genai/blob/06ec68e722c45a7218e23ea1bc1339fe4e21ecae/model/gen-ai/registry.yaml#L545
+// object or its JSON string (open-telemetry/semantic-conventions-genai@06ec68e, model/gen-ai/registry.yaml).
 // Written by: opentelemetry-instrumentation-langchain, traceloop @tool, OpenInference TOOL spans and
 // Vercel ai.toolCall.* through the span mapper.
-// Envelopes unwrapped: LangChain's on_tool_start and on_tool_end payloads, {input_str, tags, metadata,
-// inputs, kwargs} and {output, kwargs}:
-// https://github.com/traceloop/openllmetry/blob/93429cfb9767e6b51db6cb35f6b70f758117669a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/callback_handler.py#L837
-// The traceloop decorator's {args, kwargs}:
-// https://github.com/traceloop/openllmetry/blob/93429cfb9767e6b51db6cb35f6b70f758117669a/packages/traceloop-sdk/traceloop/sdk/decorators/base.py#L169
+// Envelopes unwrapped: LangChain's callback payloads, {input_str, tags, metadata, inputs, kwargs} and
+// {output, kwargs} (openllmetry callback_handler.py on_tool_start/on_tool_end), and the traceloop
+// decorator's {args, kwargs} (traceloop-sdk decorators/base.py _handle_span_input).
 
 // tool renders the call as an assistant tool_call and the result as a tool tool_call_response, both
 // carrying gen_ai.tool.call.id so the pair can be matched.

pkg/types/aiobservabilitytypes/message_format_tool_test.go

@@ -7,9 +7,8 @@ import "testing"
 func TestFormat_Tool(t *testing.T) {
 	runFormatCases(t, []formatCase{
 		{
-			// gen_ai.tool.call.arguments and gen_ai.tool.call.result written by the langchain instrumentation itself,
-			// wrapped in its on_tool_start and on_tool_end payloads:
-			// https://github.com/traceloop/openllmetry/blob/93429cfb9767e6b51db6cb35f6b70f758117669a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/callback_handler.py#L837
+			// langchain capture, execute_tool get_weather span; opentelemetry-instrumentation-langchain
+			// callback_handler.py on_tool_start/on_tool_end envelopes
 			name: "LangChainEnvelopes_InputsAndOutputUnwrapped",
 			attributes: map[string]any{
 				GenAIToolName:          "get_weather",
@@ -20,9 +19,8 @@ func TestFormat_Tool(t *testing.T) {
 			wantOutput: `[{"role": "tool", "parts": [{"type": "tool_call_response", "response": {"city": "Bengaluru", "temp_c": 18, "summary": "Clear"}}]}]`,
 		},
 		{
-			// traceloop.entity.input and traceloop.entity.output from the @tool decorator, copied by the mapper;
-			// the tool returned a JSON string, which json.dumps encoded again:
-			// https://github.com/traceloop/openllmetry/blob/93429cfb9767e6b51db6cb35f6b70f758117669a/packages/traceloop-sdk/traceloop/sdk/decorators/base.py#L169
+			// openllmetry capture, get_weather.tool span; traceloop-sdk decorators/base.py
+			// _handle_span_input {args, kwargs} and the result encoded as a JSON string twice
 			name: "TraceloopDecorator_KwargsUnwrapped_ResultDecodedTwice",
 			attributes: map[string]any{
 				GenAIToolName:          "get_weather",
@@ -33,8 +31,8 @@ func TestFormat_Tool(t *testing.T) {
 			wantOutput: `[{"role": "tool", "parts": [{"type": "tool_call_response", "response": {"city": "Bengaluru", "temp_c": 28, "summary": "Partly cloudy"}}]}]`,
 		},
 		{
-			// traceloop.entity.output of a @tool returning None, json.dumps(None), copied by the mapper:
-			// https://github.com/traceloop/openllmetry/blob/93429cfb9767e6b51db6cb35f6b70f758117669a/packages/traceloop-sdk/traceloop/sdk/decorators/base.py#L185
+			// a decorated function returning None is json.dumps(None); the openllmetry capture shows it on
+			// workflow spans, with this positional-args input
 			name: "TraceloopDecorator_PositionalArgsOnly_ListKept",
 			attributes: map[string]any{
 				GenAIToolName:          "lookup",
@@ -54,8 +52,8 @@ func TestFormat_Tool(t *testing.T) {
 			wantOutput: `[]`,
 		},
 		{
-			// input.value and output.value of an OpenInference TOOL span, copied by the mapper:
-			// https://github.com/Arize-ai/openinference/blob/12f2cc035edfb00b60b656d977f07abfb7a69900/spec/traces.md
+			// openinference-openai-agents capture, get_weather TOOL span: input.value and output.value
+			// copied by the mapper (Arize-ai/openinference@12f2cc03, spec/traces.md)
 			name: "OpenInferenceToolSpan_PlainJSONAndText",
 			attributes: map[string]any{
 				GenAIToolName:          "get_weather",
@@ -66,8 +64,8 @@ func TestFormat_Tool(t *testing.T) {
 			wantOutput: `[{"role": "tool", "parts": [{"type": "tool_call_response", "response": "27C, partly cloudy, light breeze from the south."}]}]`,
 		},
 		{
-			// ai.toolCall.{id,name,args,result} of a Vercel ai.toolCall span, copied by the mapper:
-			// https://ai-sdk.dev/docs/ai-sdk-core/telemetry
+			// vercel capture, ai.toolCall span: ai.toolCall.{id,name,args,result} copied by the mapper
+			// (https://ai-sdk.dev/docs/ai-sdk-core/telemetry)
 			name: "VercelToolCall_IDOnBothParts",
 			attributes: map[string]any{
 				GenAIToolName:          "get_current_weather",
@@ -78,6 +76,17 @@ func TestFormat_Tool(t *testing.T) {
 			wantInput:  `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "get_current_weather", "id": "call_l0ycKcmt1uPCdGWR4SkwoLBv", "arguments": {"city": "Bengaluru", "unit": "c"}}]}]`,
 			wantOutput: `[{"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_l0ycKcmt1uPCdGWR4SkwoLBv", "response": {"city": "Bengaluru", "unit": "c", "temperature": 27, "condition": "partly cloudy"}}]}]`,
 		},
+		{
+			// the object form the OTel registry asks for, whole in the JSON column
+			name: "StructuredArguments_ObjectKept",
+			attributes: map[string]any{
+				GenAIToolName:          "get_weather",
+				GenAIToolCallID:        "call_1",
+				GenAIToolCallArguments: map[string]any{"city": "Paris", "days": []any{1.0, 2.0}},
+			},
+			wantInput:  `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "get_weather", "id": "call_1", "arguments": {"city": "Paris", "days": [1, 2]}}]}]`,
+			wantOutput: `[]`,
+		},
 		{
 			// the mapper groups never fill both; the tool call wins when an SDK does
 			name: "MessagesBesideToolCall_ToolCallFormatted",

pkg/types/aiobservabilitytypes/message_format_traceloop.go

@@ -1,87 +0,0 @@
-package aiobservabilitytypes
-
-// Format: the flattened gen_ai.prompt.N.* and gen_ai.completion.N.* keys of openllmetry's instrumentations
-// from before their semconv rewrite, role, content, tool_call_id, tool_calls.M.{id,name,arguments} and
-// finish_reason, with list content as its JSON:
-// https://github.com/traceloop/openllmetry/blob/93429cfb9767e6b51db6cb35f6b70f758117669a/packages/opentelemetry-semantic-conventions-ai/opentelemetry/semconv_ai/__init__.py#L159
-// https://github.com/traceloop/openllmetry/blob/af487056a35625a2cff1c17ac83c2c0d90a02bca/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py#L408
-// https://github.com/traceloop/openllmetry/blob/af487056a35625a2cff1c17ac83c2c0d90a02bca/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py#L460
-// Written by: crewai through the openai instrumentation 0.45.6, mistralai 0.61.0 (https://github.com/traceloop/openllmetry/blob/4643b8826a6e6a4ce1968178d6ac9ca3ce607e2e/packages/opentelemetry-instrumentation-mistralai/opentelemetry/instrumentation/mistralai/__init__.py#L83),
-// litellm beside its semconv value.
-
-const (
-	traceloopPrompts     = "gen_ai.prompt"
-	traceloopCompletions = "gen_ai.completion"
-)
-
-func (f *formatting) traceloop() bool {
-	prompts := indexed(f.attributes, traceloopPrompts)
-	completions := indexed(f.attributes, traceloopCompletions)
-	if len(prompts) == 0 && len(completions) == 0 {
-		return false
-	}
-	f.out.Input = inputMessages(f.traceloopMessages(prompts))
-	f.out.Output = f.traceloopMessages(completions)
-	return true
-}
-
-// completionToolCalls fills an output message left without parts from the gen_ai.completion.N keys
-// litellm writes beside its semconv value, whose parts never carry the tool call:
-// https://github.com/BerriAI/litellm/blob/9d72bae336ec0d14c41651adcb0d2e1149721f46/litellm/integrations/opentelemetry.py#L2365
-// https://github.com/BerriAI/litellm/blob/9d72bae336ec0d14c41651adcb0d2e1149721f46/litellm/integrations/opentelemetry.py#L2770
-func (f *formatting) completionToolCalls() {
-	for i := range f.out.Output {
-		if len(f.out.Output[i].Parts) == 0 {
-			f.out.Output[i].Parts = append(f.out.Output[i].Parts, traceloopToolCalls(indexedAt(f.attributes, traceloopCompletions, i))...)
-		}
-	}
-}
-
-func (f *formatting) traceloopMessages(buckets []map[string]any) OutputMessages {
-	out := make(OutputMessages, 0, len(buckets))
-	for i, m := range buckets {
-		msg := OutputMessage{Role: role(stringOf(m["role"])), Parts: Parts{}, Name: optionalString(m["name"]), FinishReason: finishReason(stringOf(m["finish_reason"]))}
-		content, hasContent := m["content"]
-		if id := stringOf(m["tool_call_id"]); msg.Role == RoleTool && id != "" {
-			msg.Parts = Parts{part(ToolCallResponsePart{Type: PartTypeToolCallResponse, ID: &id, Response: decode(content)})}
-			out = append(out, msg)
-			continue
-		}
-		if hasContent {
-			msg.Parts = append(msg.Parts, traceloopContent(content)...)
-		}
-		msg.Parts = append(msg.Parts, traceloopToolCalls(m)...)
-		if len(msg.Parts) == 0 {
-			f.warn("message %d has no content", i)
-		}
-		out = append(out, msg)
-	}
-	return out
-}
-
-// traceloopContent reads a content value, a string or the JSON of an OpenAI content block list.
-func traceloopContent(content any) Parts {
-	if blocks, ok := decode(content).([]any); ok {
-		parts := make(Parts, 0, len(blocks))
-		for _, block := range blocks {
-			parts = append(parts, contentPart(block))
-		}
-		return parts
-	}
-	if text := stringOf(content); text != "" {
-		return Parts{part(textPart(text))}
-	}
-	return nil
-}
-
-// traceloopToolCalls reads tool_calls.M.{id,name,arguments} and litellm's function_call object.
-func traceloopToolCalls(m map[string]any) Parts {
-	var parts Parts
-	for _, call := range indexed(m, "tool_calls") {
-		parts = append(parts, part(ToolCallRequestPart{Type: PartTypeToolCall, Name: stringOf(call["name"]), ID: optionalString(call["id"]), Arguments: decode(call["arguments"])}))
-	}
-	if call, ok := m["function_call"].(map[string]any); ok {
-		parts = append(parts, part(ToolCallRequestPart{Type: PartTypeToolCall, Name: stringOf(call["name"]), Arguments: decode(call["arguments"])}))
-	}
-	return parts
-}

pkg/types/aiobservabilitytypes/message_format_traceloop_test.go

@@ -1,106 +0,0 @@
-package aiobservabilitytypes
-
-import "testing"
-
-// Captured cases are the gen_ai.prompt.N.* and gen_ai.completion.N.* keys as the collector stored
-// them for the SigNoz/scripts crewai, mistral and litellm captures.
-func TestFormat_Traceloop(t *testing.T) {
-	runFormatCases(t, []formatCase{
-		{
-			// crewai capture, openai.chat span of a tool round, as the openai instrumentation 0.45.6 wrote
-			// prompts and completions before its semconv rewrite
-			// https://github.com/traceloop/openllmetry/blob/af487056a35625a2cff1c17ac83c2c0d90a02bca/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py#L408
-			name: "ToolRound_ToolCallAndToolMessages",
-			attributes: map[string]any{
-				"gen_ai.prompt.0.role":                   "system",
-				"gen_ai.prompt.0.content":                "You are Weather Reporter. Call get_weather and summarise the result.",
-				"gen_ai.prompt.1.role":                   "user",
-				"gen_ai.prompt.1.content":                "Current Task: What's the weather in Bengaluru?",
-				"gen_ai.prompt.2.role":                   "assistant",
-				"gen_ai.prompt.2.tool_calls.0.id":        "call_bbVgWPe8AMBol7JlAhDkA6R7",
-				"gen_ai.prompt.2.tool_calls.0.name":      "get_weather",
-				"gen_ai.prompt.2.tool_calls.0.arguments": `{"city":"Bengaluru"}`,
-				"gen_ai.prompt.3.role":                   "tool",
-				"gen_ai.prompt.3.tool_call_id":           "call_bbVgWPe8AMBol7JlAhDkA6R7",
-				"gen_ai.prompt.3.content":                `{"city": "Bengaluru", "temp_c": 18, "summary": "Clear"}`,
-				"gen_ai.completion.0.role":               "assistant",
-				"gen_ai.completion.0.finish_reason":      "stop",
-				"gen_ai.completion.0.content":            "The current weather in Bengaluru is clear with a temperature of 18°C.",
-			},
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "You are Weather Reporter. Call get_weather and summarise the result."}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "Current Task: What's the weather in Bengaluru?"}]},
-			  {"role": "assistant", "parts": [{"type": "tool_call", "name": "get_weather", "id": "call_bbVgWPe8AMBol7JlAhDkA6R7", "arguments": {"city": "Bengaluru"}}]},
-			  {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_bbVgWPe8AMBol7JlAhDkA6R7", "response": {"city": "Bengaluru", "temp_c": 18, "summary": "Clear"}}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "The current weather in Bengaluru is clear with a temperature of 18°C."}], "finish_reason": "stop"}]`,
-		},
-		{
-			// crewai capture, openai.chat span answering with two tool calls
-			// https://github.com/traceloop/openllmetry/blob/af487056a35625a2cff1c17ac83c2c0d90a02bca/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py#L460
-			name: "CompletionToolCalls_ToolCallParts",
-			attributes: map[string]any{
-				"gen_ai.prompt.0.role":                       "user",
-				"gen_ai.prompt.0.content":                    "What's the weather in Paris and in London?",
-				"gen_ai.completion.0.role":                   "assistant",
-				"gen_ai.completion.0.finish_reason":          "tool_calls",
-				"gen_ai.completion.0.tool_calls.0.id":        "call_1",
-				"gen_ai.completion.0.tool_calls.0.name":      "get_weather",
-				"gen_ai.completion.0.tool_calls.0.arguments": `{"city":"Paris"}`,
-				"gen_ai.completion.0.tool_calls.1.id":        "call_2",
-				"gen_ai.completion.0.tool_calls.1.name":      "get_weather",
-				"gen_ai.completion.0.tool_calls.1.arguments": `{"city":"London"}`,
-			},
-			wantInput: `[{"role": "user", "parts": [{"type": "text", "content": "What's the weather in Paris and in London?"}]}]`,
-			wantOutput: `[{"role": "assistant", "parts": [
-			  {"type": "tool_call", "name": "get_weather", "id": "call_1", "arguments": {"city": "Paris"}},
-			  {"type": "tool_call", "name": "get_weather", "id": "call_2", "arguments": {"city": "London"}}
-			], "finish_reason": "tool_call"}]`,
-		},
-		{
-			// mistral capture, mistralai.chat span: the instrumentation writes the role for every message but
-			// drops a content that is not a plain string
-			// https://github.com/traceloop/openllmetry/blob/4643b8826a6e6a4ce1968178d6ac9ca3ce607e2e/packages/opentelemetry-instrumentation-mistralai/opentelemetry/instrumentation/mistralai/__init__.py#L75
-			// https://github.com/traceloop/openllmetry/blob/4643b8826a6e6a4ce1968178d6ac9ca3ce607e2e/packages/opentelemetry-instrumentation-mistralai/opentelemetry/instrumentation/mistralai/__init__.py#L88
-			name: "RoleWithoutContent_EmptyMessageWarns",
-			attributes: map[string]any{
-				"gen_ai.prompt.0.role":              "user",
-				"gen_ai.completion.0.role":          "assistant",
-				"gen_ai.completion.0.finish_reason": "stop",
-				"gen_ai.completion.0.content":       "Observability is the ability to understand a system's internal state by analyzing its external outputs.",
-			},
-			warnings:   []string{"message 0 has no content"},
-			wantInput:  `[{"role": "user", "parts": []}]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "Observability is the ability to understand a system's internal state by analyzing its external outputs."}], "finish_reason": "stop"}]`,
-		},
-		{
-			// a vision prompt: list content is written as its JSON, the OpenAI content block list
-			// https://github.com/traceloop/openllmetry/blob/af487056a35625a2cff1c17ac83c2c0d90a02bca/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py#L431
-			name: "ContentBlockList_TextAndBlobParts",
-			attributes: map[string]any{
-				"gen_ai.prompt.0.role":    "user",
-				"gen_ai.prompt.0.content": `[{"type":"text","text":"What animal is this?"},{"type":"image_url","image_url":{"url":"data:image/jpeg;base64,/9j/4AAQ"}}]`,
-			},
-			wantInput: `[{"role": "user", "parts": [
-			  {"type": "text", "content": "What animal is this?"},
-			  {"type": "blob", "modality": "image", "mime_type": "image/jpeg", "content": "/9j/4AAQ"}
-			]}]`,
-			wantOutput: `[]`,
-		},
-		{
-			// litellm capture, litellm_request span: the semconv output has no part for the tool call, the
-			// gen_ai.completion.0.function_call keys beside it still carry it
-			// https://github.com/BerriAI/litellm/blob/9d72bae336ec0d14c41651adcb0d2e1149721f46/litellm/integrations/opentelemetry.py#L2365
-			// https://github.com/BerriAI/litellm/blob/9d72bae336ec0d14c41651adcb0d2e1149721f46/litellm/integrations/opentelemetry.py#L2770
-			name: "SemconvEmptyParts_ToolCallFromCompletionKeys",
-			attributes: map[string]any{
-				GenAIInputMessages:                            `[{"role": "user", "parts": [{"type": "text", "content": "What is the weather in Bengaluru in celsius?"}]}]`,
-				GenAIOutputMessages:                           `[{"role": "assistant", "parts": [], "finish_reason": "tool_calls"}]`,
-				"gen_ai.completion.0.function_call.name":      "get_current_weather",
-				"gen_ai.completion.0.function_call.arguments": `{"city":"Bengaluru"}`,
-			},
-			wantInput:  `[{"role": "user", "parts": [{"type": "text", "content": "What is the weather in Bengaluru in celsius?"}]}]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "get_current_weather", "arguments": {"city": "Bengaluru"}}], "finish_reason": "tool_call"}]`,
-		},
-	})
-}

pkg/types/aiobservabilitytypes/message_format_vercel.go

@@ -1,189 +0,0 @@
-package aiobservabilitytypes
-
-import (
-	"encoding/base64"
-	"strings"
-)
-
-// Format: the AI SDK's legacy telemetry attributes. ai.prompt.messages on the doGenerate and doStream
-// spans is the provider prompt, ai.prompt on the parent generateText and streamText spans is {system,
-// prompt} or {messages}, ai.response.text and ai.response.toolCalls hold the answer:
-// https://ai-sdk.dev/docs/ai-sdk-core/telemetry
-// The prompt parts in the provider's v1 spelling, image with mimeType, tool-call args and tool-result
-// result, or v2, file with mediaType, tool-call input and tool-result output:
-// https://github.com/vercel/ai/blob/47bd16dd50283c82a5fef4c839d7f4102c45f5b9/packages/provider/src/language-model/v1/language-model-v1-prompt.ts#L120
-// https://github.com/vercel/ai/blob/a5e92fe3c692f8e152fe1eb9ef50a4e734ad0ffb/packages/provider/src/language-model/v2/language-model-v2-prompt.ts#L93
-// The span mapper moves ai.prompt.messages and one of the response keys onto gen_ai.*.messages; the other
-// response key and ai.prompt are read where they are.
-// Written by: the Vercel AI SDK, Mastra.
-// Handled: text, reasoning, tool-call, tool-result, image and file parts as a data URL, a URL, bare base64
-// or a serialised Buffer; ai.response.finishReason.
-
-const (
-	vercelPrompt           = "ai.prompt"
-	vercelOperation        = "ai.operationId"
-	vercelResponseText     = "ai.response.text"
-	vercelResponseCalls    = "ai.response.toolCalls"
-	vercelResponseFinished = "ai.response.finishReason"
-)
-
-// vercel detects the AI SDK by ai.operationId, which every span it emits carries:
-// https://github.com/vercel/ai/blob/a5e92fe3c692f8e152fe1eb9ef50a4e734ad0ffb/packages/ai/src/telemetry/assemble-operation-name.ts#L18
-func (f *formatting) vercel() bool {
-	if attribute(f.attributes, vercelOperation) == nil {
-		return false
-	}
-	switch input := decode(attribute(f.attributes, GenAIInputMessages)).(type) {
-	case []any:
-		f.out.Input = inputMessages(vercelMessages(input))
-	case nil:
-		if prompt, ok := decode(attribute(f.attributes, vercelPrompt)).(map[string]any); ok {
-			f.out.Input = inputMessages(vercelParentPrompt(prompt))
-		}
-	default:
-		f.out.Input = inputMessages(OutputMessages{genericMessage(input)})
-		f.warn("input is not a message list, kept as generic")
-	}
-
-	text, calls := attribute(f.attributes, vercelResponseText), decode(attribute(f.attributes, vercelResponseCalls))
-	switch output := decode(attribute(f.attributes, GenAIOutputMessages)).(type) {
-	case string:
-		text = output
-	case []any:
-		calls = output
-	}
-	msg := OutputMessage{Role: RoleAssistant, Parts: Parts{}, FinishReason: finishReason(stringOf(attribute(f.attributes, vercelResponseFinished)))}
-	if s := stringOf(text); s != "" {
-		msg.Parts = append(msg.Parts, part(textPart(s)))
-	}
-	if list, ok := calls.([]any); ok {
-		for _, item := range list {
-			if call, ok := item.(map[string]any); ok {
-				msg.Parts = append(msg.Parts, vercelToolCall(call))
-			}
-		}
-	}
-	if len(msg.Parts) > 0 {
-		f.out.Output = OutputMessages{msg}
-	}
-	return true
-}
-
-// vercelParentPrompt reads the parent span's ai.prompt: the system prompt with either a bare prompt or
-// the messages.
-func vercelParentPrompt(prompt map[string]any) OutputMessages {
-	var msgs OutputMessages
-	if parts := systemParts(prompt["system"]); len(parts) > 0 {
-		msgs = append(msgs, OutputMessage{Role: RoleSystem, Parts: parts})
-	}
-	if messages, ok := prompt["messages"].([]any); ok {
-		return append(msgs, vercelMessages(messages)...)
-	}
-	if text := stringOf(prompt["prompt"]); text != "" {
-		msgs = append(msgs, OutputMessage{Role: RoleUser, Parts: Parts{part(textPart(text))}})
-	}
-	return msgs
-}
-
-func vercelMessages(list []any) OutputMessages {
-	out := make(OutputMessages, 0, len(list))
-	for _, item := range list {
-		m, ok := item.(map[string]any)
-		if !ok {
-			out = append(out, genericMessage(item))
-			continue
-		}
-		msg := OutputMessage{Role: role(stringOf(m["role"])), Parts: Parts{}}
-		switch content := m["content"].(type) {
-		case string:
-			if content != "" {
-				msg.Parts = append(msg.Parts, part(textPart(content)))
-			}
-		case []any:
-			for _, item := range content {
-				msg.Parts = append(msg.Parts, vercelPart(item))
-			}
-		}
-		out = append(out, msg)
-	}
-	return out
-}
-
-func vercelPart(item any) Part {
-	p, ok := item.(map[string]any)
-	if !ok {
-		return contentPart(item)
-	}
-	switch stringOf(p["type"]) {
-	case "text":
-		return part(textPart(stringOf(p["text"])))
-	case "reasoning":
-		return part(ReasoningPart{Type: PartTypeReasoning, Content: stringOf(p["text"])})
-	case "redacted-reasoning":
-		return part(ReasoningPart{Type: PartTypeReasoning})
-	case "tool-call":
-		return vercelToolCall(p)
-	case "tool-result":
-		response := p["result"]
-		if output, ok := p["output"].(map[string]any); ok {
-			response = output["value"]
-		}
-		return part(ToolCallResponsePart{Type: PartTypeToolCallResponse, ID: optionalString(p["toolCallId"]), Response: response})
-	case "image":
-		mime := stringOf(p["mimeType"])
-		if mime == "" {
-			mime = stringOf(p["mediaType"])
-		}
-		if media, ok := vercelMedia(p["image"], mime, ModalityImage); ok {
-			return media
-		}
-	case "file":
-		mime := stringOf(p["mediaType"])
-		if mime == "" {
-			mime = stringOf(p["mimeType"])
-		}
-		if media, ok := vercelMedia(p["data"], mime, modalityOf(mime)); ok {
-			return media
-		}
-	}
-	return part(GenericPart(p))
-}
-
-// vercelToolCall reads {toolCallId, toolName, input} or the v1 {toolCallId, toolName, args}; the
-// arguments may be a JSON string:
-// https://github.com/vercel/ai/blob/a5e92fe3c692f8e152fe1eb9ef50a4e734ad0ffb/packages/provider/src/language-model/v2/language-model-v2-prompt.ts#L126
-// https://github.com/vercel/ai/blob/47bd16dd50283c82a5fef4c839d7f4102c45f5b9/packages/provider/src/language-model/v1/language-model-v1-prompt.ts#L175
-func vercelToolCall(call map[string]any) Part {
-	args := call["input"]
-	if args == nil {
-		args = call["args"]
-	}
-	return part(ToolCallRequestPart{Type: PartTypeToolCall, Name: stringOf(call["toolName"]), ID: optionalString(call["toolCallId"]), Arguments: decode(args)})
-}
-
-// vercelMedia reads a data URL, a URL, bare base64 or a serialised Node Buffer {type: Buffer,
-// data: [bytes]}, which is how the parent span's ai.prompt carries an image given as bytes.
-func vercelMedia(value any, mimeType string, modality Modality) (Part, bool) {
-	switch v := value.(type) {
-	case string:
-		if v == "" {
-			return Part{}, false
-		}
-		if strings.Contains(v, "://") || strings.HasPrefix(v, "data:") {
-			return part(mediaPart(v, modality)), true
-		}
-		return part(BlobPart{Type: PartTypeBlob, Modality: modality, Content: v, MimeType: optionalString(mimeType)}), true
-	case map[string]any:
-		bytes, ok := v["data"].([]any)
-		if !ok {
-			return Part{}, false
-		}
-		raw := make([]byte, 0, len(bytes))
-		for _, b := range bytes {
-			n, _ := b.(float64)
-			raw = append(raw, byte(n))
-		}
-		return part(BlobPart{Type: PartTypeBlob, Modality: modality, Content: base64.StdEncoding.EncodeToString(raw), MimeType: optionalString(mimeType)}), true
-	}
-	return Part{}, false
-}

pkg/types/aiobservabilitytypes/message_format_vercel_test.go

@@ -1,159 +0,0 @@
-package aiobservabilitytypes
-
-import "testing"
-
-// Captured cases are the ai.* attributes as the collector stored them for the SigNoz/scripts vercel and
-// mastra captures, the AI SDK v4 shapes; the v5 cases follow the provider's v2 prompt types at the
-// linked lines.
-func TestFormat_Vercel(t *testing.T) {
-	runFormatCases(t, []formatCase{
-		{
-			// vercel capture, ai.generateText.doGenerate span after a tool round: v1 tool-call args and
-			// tool-result result
-			// https://github.com/vercel/ai/blob/47bd16dd50283c82a5fef4c839d7f4102c45f5b9/packages/provider/src/language-model/v1/language-model-v1-prompt.ts#L175
-			name: "V4ToolRound_ToolCallAndResultParts",
-			attributes: map[string]any{
-				"ai.operationId":           "ai.generateText.doGenerate",
-				"ai.response.finishReason": "stop",
-				GenAIInputMessages:         `[{"role":"user","content":[{"type":"text","text":"What is the weather in Paris and in London, in celsius? Use the tool for each."}]},{"role":"assistant","content":[{"type":"tool-call","toolCallId":"call_isBPb3o3cqhY8OpugQntLOgy","toolName":"get_current_weather","args":{"city":"Paris","unit":"c"}},{"type":"tool-call","toolCallId":"call_sH9xd5EWFkiSaWmYvg4CeWTl","toolName":"get_current_weather","args":{"city":"London","unit":"c"}}]},{"role":"tool","content":[{"type":"tool-result","toolCallId":"call_isBPb3o3cqhY8OpugQntLOgy","toolName":"get_current_weather","result":{"city":"Paris","unit":"c","temperature":18,"condition":"clear"}},{"type":"tool-result","toolCallId":"call_sH9xd5EWFkiSaWmYvg4CeWTl","toolName":"get_current_weather","result":{"city":"London","unit":"c","temperature":18,"condition":"clear"}}]}]`,
-				GenAIOutputMessages:        "The current weather is as follows:\n\n- **Paris**: 18°C, clear\n- **London**: 18°C, clear",
-			},
-			wantInput: `[
-			  {"role": "user", "parts": [{"type": "text", "content": "What is the weather in Paris and in London, in celsius? Use the tool for each."}]},
-			  {"role": "assistant", "parts": [
-			    {"type": "tool_call", "name": "get_current_weather", "id": "call_isBPb3o3cqhY8OpugQntLOgy", "arguments": {"city": "Paris", "unit": "c"}},
-			    {"type": "tool_call", "name": "get_current_weather", "id": "call_sH9xd5EWFkiSaWmYvg4CeWTl", "arguments": {"city": "London", "unit": "c"}}
-			  ]},
-			  {"role": "tool", "parts": [
-			    {"type": "tool_call_response", "id": "call_isBPb3o3cqhY8OpugQntLOgy", "response": {"city": "Paris", "unit": "c", "temperature": 18, "condition": "clear"}},
-			    {"type": "tool_call_response", "id": "call_sH9xd5EWFkiSaWmYvg4CeWTl", "response": {"city": "London", "unit": "c", "temperature": 18, "condition": "clear"}}
-			  ]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "The current weather is as follows:\n\n- **Paris**: 18°C, clear\n- **London**: 18°C, clear"}], "finish_reason": "stop"}]`,
-		},
-		{
-			// vercel capture, ai.streamText.doStream span: the mapper moved ai.response.toolCalls onto the output
-			// and left the empty ai.response.text behind
-			name: "V4ToolCalls_ToolCallParts_EmptyTextIgnored",
-			attributes: map[string]any{
-				"ai.operationId":           "ai.streamText.doStream",
-				"ai.response.finishReason": "tool-calls",
-				"ai.response.text":         "",
-				GenAIInputMessages:         `[{"role":"user","content":[{"type":"text","text":"What is the weather in Bengaluru in celsius?"}]}]`,
-				GenAIOutputMessages:        `[{"type":"tool-call","toolCallId":"call_l0ycKcmt1uPCdGWR4SkwoLBv","toolName":"get_current_weather","args":{"city":"Bengaluru","unit":"c"}}]`,
-			},
-			wantInput:  `[{"role": "user", "parts": [{"type": "text", "content": "What is the weather in Bengaluru in celsius?"}]}]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "get_current_weather", "id": "call_l0ycKcmt1uPCdGWR4SkwoLBv", "arguments": {"city": "Bengaluru", "unit": "c"}}], "finish_reason": "tool_call"}]`,
-		},
-		{
-			// vercel capture, ai.generateText parent span: ai.prompt is unmapped, the answer text was moved
-			name: "ParentPromptWithSystem_SystemAndUserMessages",
-			attributes: map[string]any{
-				"ai.operationId":           "ai.generateText",
-				"ai.prompt":                `{"system":"You are concise.","prompt":"Say hello in five words."}`,
-				"ai.response.finishReason": "stop",
-				GenAIOutputMessages:        "Hello! How are you today?",
-			},
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "You are concise."}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "Say hello in five words."}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "Hello! How are you today?"}], "finish_reason": "stop"}]`,
-		},
-		{
-			// vercel capture, ai.generateText parent span of a vision call: the image is a serialised Node Buffer
-			name: "ParentPromptMessages_BufferImage_BlobPart",
-			attributes: map[string]any{
-				"ai.operationId":    "ai.generateText",
-				"ai.prompt":         `{"messages":[{"role":"user","content":[{"type":"text","text":"What animal is in this image? One sentence."},{"type":"image","image":{"type":"Buffer","data":[137,80,78,71]},"mediaType":"image/png"}]}]}`,
-				GenAIOutputMessages: "A duck.",
-			},
-			wantInput: `[{"role": "user", "parts": [
-			  {"type": "text", "content": "What animal is in this image? One sentence."},
-			  {"type": "blob", "modality": "image", "mime_type": "image/png", "content": "iVBORw=="}
-			]}]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "A duck."}]}]`,
-		},
-		{
-			// the v2 prompt after a tool round: tool-call input and tool-result output
-			// https://github.com/vercel/ai/blob/a5e92fe3c692f8e152fe1eb9ef50a4e734ad0ffb/packages/provider/src/language-model/v2/language-model-v2-prompt.ts#L126
-			// https://github.com/vercel/ai/blob/a5e92fe3c692f8e152fe1eb9ef50a4e734ad0ffb/packages/provider/src/language-model/v2/language-model-v2-prompt.ts#L161
-			name: "V5ToolRound_InputAndOutputSpellings",
-			attributes: map[string]any{
-				"ai.operationId":           "ai.generateText.doGenerate",
-				"ai.response.finishReason": "stop",
-				GenAIInputMessages:         `[{"role":"system","content":"Use the tool."},{"role":"user","content":[{"type":"text","text":"Weather in Paris?"}]},{"role":"assistant","content":[{"type":"tool-call","toolCallId":"call_1","toolName":"getWeather","input":{"city":"Paris"}}]},{"role":"tool","content":[{"type":"tool-result","toolCallId":"call_1","toolName":"getWeather","output":{"type":"json","value":{"city":"Paris","tempC":18,"summary":"Clear"}}}]}]`,
-				GenAIOutputMessages:        "Paris is clear at 18C.",
-			},
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "Use the tool."}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "Weather in Paris?"}]},
-			  {"role": "assistant", "parts": [{"type": "tool_call", "name": "getWeather", "id": "call_1", "arguments": {"city": "Paris"}}]},
-			  {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_1", "response": {"city": "Paris", "tempC": 18, "summary": "Clear"}}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "text", "content": "Paris is clear at 18C."}], "finish_reason": "stop"}]`,
-		},
-		{
-			// ai.response.toolCalls in the v2 spelling, the arguments under input; the text the mapper left
-			// behind in ai.response.text comes first
-			// https://github.com/vercel/ai/blob/a5e92fe3c692f8e152fe1eb9ef50a4e734ad0ffb/packages/provider/src/language-model/v2/language-model-v2-prompt.ts#L126
-			name: "V5ToolCallsBesideText_TextThenToolCall",
-			attributes: map[string]any{
-				"ai.operationId":           "ai.streamText.doStream",
-				"ai.response.finishReason": "tool-calls",
-				"ai.response.text":         "Let me check.",
-				GenAIInputMessages:         `[{"role":"user","content":[{"type":"text","text":"Weather in Paris? Use the tool."}]}]`,
-				GenAIOutputMessages:        `[{"toolCallId":"call_2","toolName":"getWeather","input":{"city":"Paris"}}]`,
-			},
-			wantInput: `[{"role": "user", "parts": [{"type": "text", "content": "Weather in Paris? Use the tool."}]}]`,
-			wantOutput: `[{"role": "assistant", "parts": [
-			  {"type": "text", "content": "Let me check."},
-			  {"type": "tool_call", "name": "getWeather", "id": "call_2", "arguments": {"city": "Paris"}}
-			], "finish_reason": "tool_call"}]`,
-		},
-		{
-			// the v2 file part, data as bare base64 or a URL with its mediaType, a reasoning part, and a tool
-			// result given as text
-			// https://github.com/vercel/ai/blob/a5e92fe3c692f8e152fe1eb9ef50a4e734ad0ffb/packages/provider/src/language-model/v2/language-model-v2-prompt.ts#L93
-			// https://github.com/vercel/ai/blob/a5e92fe3c692f8e152fe1eb9ef50a4e734ad0ffb/packages/provider/src/language-model/v2/language-model-v2-prompt.ts#L187
-			name: "V5FileParts_BlobsByMediaType_URLAsUri",
-			attributes: map[string]any{
-				"ai.operationId":   "ai.generateText.doGenerate",
-				GenAIInputMessages: `[{"role":"user","content":[{"type":"text","text":"What is in these?"},{"type":"file","mediaType":"image/png","data":"iVBORw0KGgo="},{"type":"file","mediaType":"application/pdf","filename":"a.pdf","data":"JVBERi0xLjQgc3R1Yg=="},{"type":"file","mediaType":"audio/wav","data":"https://example.com/a.wav"},{"type":"reasoning","text":"thinking"}]},{"role":"tool","content":[{"type":"tool-result","toolCallId":"call_1","toolName":"read","output":{"type":"text","value":"done"}}]}]`,
-			},
-			wantInput: `[
-			  {"role": "user", "parts": [
-			    {"type": "text", "content": "What is in these?"},
-			    {"type": "blob", "modality": "image", "mime_type": "image/png", "content": "iVBORw0KGgo="},
-			    {"type": "blob", "modality": "document", "mime_type": "application/pdf", "content": "JVBERi0xLjQgc3R1Yg=="},
-			    {"type": "uri", "modality": "audio", "uri": "https://example.com/a.wav"},
-			    {"type": "reasoning", "content": "thinking"}
-			  ]},
-			  {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_1", "response": "done"}]}
-			]`,
-			wantOutput: `[]`,
-		},
-		{
-			// mastra capture, ai.generateText.doGenerate span: the v4 shapes with the arguments as a JSON string
-			name: "MastraToolCalls_ToolCallPart",
-			attributes: map[string]any{
-				"ai.operationId":           "ai.generateText.doGenerate",
-				"ai.response.finishReason": "tool-calls",
-				GenAIInputMessages:         `[{"role":"system","content":"You are a weather assistant. When asked about weather, call get-weather once per city and summarise."},{"role":"user","content":[{"type":"text","text":"What is the weather in Bengaluru?"}]}]`,
-				GenAIOutputMessages:        `[{"toolCallType":"function","toolCallId":"call_526phoykytobsvLouzES805J","toolName":"getWeather","args":"{\"city\":\"Bengaluru\"}"}]`,
-			},
-			wantInput: `[
-			  {"role": "system", "parts": [{"type": "text", "content": "You are a weather assistant. When asked about weather, call get-weather once per city and summarise."}]},
-			  {"role": "user", "parts": [{"type": "text", "content": "What is the weather in Bengaluru?"}]}
-			]`,
-			wantOutput: `[{"role": "assistant", "parts": [{"type": "tool_call", "name": "getWeather", "id": "call_526phoykytobsvLouzES805J", "arguments": {"city": "Bengaluru"}}], "finish_reason": "tool_call"}]`,
-		},
-		{
-			// vercel capture, ai.generateText span of a failed call
-			name:       "PromptOnlyNoResponse_InputOnly",
-			attributes: map[string]any{"ai.operationId": "ai.generateText", "ai.prompt": `{"prompt":"this should fail"}`},
-			wantInput:  `[{"role": "user", "parts": [{"type": "text", "content": "this should fail"}]}]`,
-			wantOutput: `[]`,
-		},
-	})
-}

pkg/types/dashboardtypes/perses_dashboard_data.go

@@ -123,10 +123,22 @@ func (d *DashboardSpec) validatePanels() error {
 				return err
 			}
 		}
+		// validatePanelQueryCount left exactly one query for a spec that reads it.
+		if spec, ok := panel.Spec.Plugin.Spec.(queryBoundPanelSpec); ok {
+			if err := spec.validateAgainstQuery(panel.Spec.Queries[0], path+".spec.plugin.spec"); err != nil {
+				return err
+			}
+		}
 	}
 	return nil
 }
 
+// queryBoundPanelSpec is a panel spec with fields that depend on the panel's
+// query. It is checked after the query itself validates.
+type queryBoundPanelSpec interface {
+	validateAgainstQuery(query Query, path string) error
+}
+
 func validatePanelQueryCount(queries []Query, panelKind PanelPluginKind, path string) error {
 	if queries == nil {
 		return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "%s.spec.queries: is required and must not be null; use [] for a panel that renders without a query", path)

pkg/types/dashboardtypes/perses_dashboard_test.go

@@ -1870,6 +1870,319 @@ func TestInvalidateAreaChartPanelSpecValues(t *testing.T) {
 	}
 }
 
+func TestScatterPlotPanelDefaults(t *testing.T) {
+	data := []byte(`{
+		"variables": [],
+		"panels": {
+			"p1": {
+				"kind": "Panel",
+				"spec": {
+					"links": [],
+					"plugin": {
+						"kind": "signoz/ScatterPlotPanel",
+						"spec": {}
+					},
+					"queries": [{"kind": "scalar", "spec": {"plugin": {"kind": "signoz/ClickHouseSQL", "spec": {"name": "A", "query": "SELECT 1"}}}}]
+				}
+			}
+		},
+		"links": [],
+		"layouts": []
+	}`)
+	d, err := unmarshalDashboard(data)
+	require.NoError(t, err, "unmarshal and validate failed")
+
+	require.IsType(t, &ScatterPlotPanelSpec{}, d.Panels["p1"].Spec.Plugin.Spec)
+	spec := d.Panels["p1"].Spec.Plugin.Spec.(*ScatterPlotPanelSpec)
+
+	assert.Empty(t, spec.Dimensions.X, "an omitted x stays empty so the renderer picks the first value column")
+	assert.Empty(t, spec.Dimensions.Y, "an omitted y stays empty so the renderer picks the second value column")
+	assert.Empty(t, spec.Dimensions.ColorBy, "an omitted colorBy stays empty so the renderer colours by every group-by label")
+	assert.Equal(t, "auto", spec.Axes.X.Scale.ValueOrDefault(), "expected x scale default auto")
+	assert.Equal(t, "auto", spec.Axes.Y.Scale.ValueOrDefault(), "expected y scale default auto")
+	assert.Nil(t, spec.Axes.X.SoftMin, "expected x softMin unset")
+	assert.Equal(t, "2", spec.Formatting.DecimalPrecision.ValueOrDefault(), "expected DecimalPrecision default 2")
+	assert.Equal(t, "global_time", spec.Visualization.TimePreference.ValueOrDefault(), "expected TimePreference default global_time")
+	assert.Equal(t, "bottom", spec.Legend.Position.ValueOrDefault(), "expected LegendPosition default bottom")
+
+	output, err := json.Marshal(d)
+	require.NoError(t, err, "marshal dashboard failed")
+	assert.Contains(t, string(output), `"axes":{"x":{"softMin":null,"softMax":null,"scale":"auto","label":""},"y":{"softMin":null,"softMax":null,"scale":"auto","label":""}}`, "expected default axes in stored/response JSON")
+	assert.Contains(t, string(output), `"dimensions":{"x":"","y":"","sizeBy":"","colorBy":null}`, "expected empty dimensions in stored/response JSON")
+	assert.Contains(t, string(output), `"chartAppearance":{"points":{"size":null,"minSize":null,"maxSize":null,"opacity":null}}`, "omitted point settings stay null so the renderer applies its defaults")
+}
+
+func TestValidateScatterPlotDashboard(t *testing.T) {
+	data, err := os.ReadFile("testdata/perses_scatter_plot_panel.json")
+	require.NoError(t, err, "reading example file")
+
+	spec, err := unmarshalDashboard(data)
+	require.NoError(t, err, "unmarshal and validate failed")
+
+	require.IsType(t, &ScatterPlotPanelSpec{}, spec.Panels["p1"].Spec.Plugin.Spec)
+	panelSpec := spec.Panels["p1"].Spec.Plugin.Spec.(*ScatterPlotPanelSpec)
+	softMin, softMax := 0.0, 1000.0
+	size, minSize, maxSize, opacity := PointDiameter(8), PointDiameter(4), PointDiameter(24), PointOpacity(0.1)
+	assert.Equal(t, &ScatterPlotPanelSpec{
+		Visualization: BasicVisualization{TimePreference: TimePreferenceLast1Hr},
+		Dimensions:    ScatterPlotDimensions{X: "A.count()", Y: "A.p99(duration_nano)", SizeBy: "A.countIf(has_error = true)", ColorBy: []string{"k8s.namespace.name", "k8s.pod.name"}},
+		Formatting:    TableFormatting{ColumnUnits: map[string]string{"A.p99(duration_nano)": "ns"}, DecimalPrecision: PrecisionOption1},
+		Axes: ScatterPlotAxes{
+			X: ScatterPlotAxis{SoftMin: &softMin, Scale: AxisScaleLog, Label: "Throughput"},
+			Y: ScatterPlotAxis{SoftMax: &softMax, Scale: AxisScaleSymlog, Label: "p99 latency"},
+		},
+		ChartAppearance: ScatterPlotChartAppearance{Points: ScatterPlotPoints{Size: &size, MinSize: &minSize, MaxSize: &maxSize, Opacity: &opacity}},
+		Legend:          Legend{Position: LegendPositionRight, Mode: LegendModeList, CustomColors: map[string]string{"checkout": "#1f77b4"}},
+		Thresholds: []ScatterPlotThreshold{
+			{ThresholdWithLabel: ThresholdWithLabel{Value: 300, Unit: "ms", Color: "#f00", Label: "p99 SLO"}, Axis: ThresholdAxisY},
+			{ThresholdWithLabel: ThresholdWithLabel{Value: 50, Color: "#0f0"}, Axis: ThresholdAxisX},
+		},
+	}, panelSpec)
+
+	dashboard := &DashboardV2{Spec: *spec}
+	request, err := dashboard.GetPanelQuery(1, 2, "p1")
+	require.NoError(t, err, "building the panel's query failed")
+	assert.Equal(t, qb.RequestTypeScalar, request.RequestType)
+	require.NotNil(t, request.FormatOptions)
+	assert.True(t, request.FormatOptions.FormatTableResultForUI, "scatter rows are joined the way a table's are")
+
+	require.Len(t, request.CompositeQuery.Queries, 1)
+	query, ok := request.CompositeQuery.Queries[0].Spec.(qb.QueryBuilderQuery[qb.TraceAggregation])
+	require.True(t, ok, "expected a traces builder query")
+	assert.Len(t, query.Aggregations, 3)
+	assert.Len(t, query.GroupBy, 2)
+
+	require.NoError(t, request.Validate(), "the request built from the panel is not a valid scalar request")
+
+	// the panel read back out of storage draws the same scatter plot
+	stored, err := json.Marshal(spec)
+	require.NoError(t, err, "marshal dashboard failed")
+	reread, err := unmarshalDashboard(stored)
+	require.NoError(t, err, "the stored dashboard does not validate")
+	assert.Equal(t, panelSpec, reread.Panels["p1"].Spec.Plugin.Spec)
+	rereadRequest, err := (&DashboardV2{Spec: *reread}).GetPanelQuery(1, 2, "p1")
+	require.NoError(t, err, "building the stored panel's query failed")
+	assert.Equal(t, request, rereadRequest)
+}
+
+func TestInvalidateScatterPlotPanelSpecValues(t *testing.T) {
+	tests := []struct {
+		scenario               string
+		panelKind              string
+		panelSpec              string
+		expectedErrorSubstring string
+	}{
+		{
+			scenario:               "unknown axis scale",
+			panelKind:              "signoz/ScatterPlotPanel",
+			panelSpec:              `{"axes": {"x": {"scale": "sqrt"}}}`,
+			expectedErrorSubstring: "invalid axis scale \"sqrt\": must be `auto`, `linear`, `log`, or `symlog`",
+		},
+		{
+			scenario:               "non-string axis scale",
+			panelKind:              "signoz/ScatterPlotPanel",
+			panelSpec:              `{"axes": {"y": {"scale": true}}}`,
+			expectedErrorSubstring: "cannot unmarshal bool",
+		},
+		{
+			scenario:               "isLogScale on a scatter plot axis",
+			panelKind:              "signoz/ScatterPlotPanel",
+			panelSpec:              `{"axes": {"x": {"isLogScale": true}}}`,
+			expectedErrorSubstring: "unknown field",
+		},
+		{
+			scenario:               "single-axis axes shape on a scatter plot panel",
+			panelKind:              "signoz/ScatterPlotPanel",
+			panelSpec:              `{"axes": {"softMin": 0}}`,
+			expectedErrorSubstring: "unknown field",
+		},
+		{
+			scenario:               "unknown threshold axis",
+			panelKind:              "signoz/ScatterPlotPanel",
+			panelSpec:              `{"thresholds": [{"value": 1, "color": "#f00", "axis": "z"}]}`,
+			expectedErrorSubstring: "invalid threshold axis \"z\": must be `x` or `y`",
+		},
+		{
+			scenario:               "unknown dimension",
+			panelKind:              "signoz/ScatterPlotPanel",
+			panelSpec:              `{"dimensions": {"z": "A"}}`,
+			expectedErrorSubstring: "unknown field",
+		},
+		{
+			scenario:               "point size below the minimum",
+			panelKind:              "signoz/ScatterPlotPanel",
+			panelSpec:              `{"chartAppearance": {"points": {"size": 1}}}`,
+			expectedErrorSubstring: "invalid point diameter 1: must be a whole number between 2 and 40",
+		},
+		{
+			scenario:               "max point size above the maximum",
+			panelKind:              "signoz/ScatterPlotPanel",
+			panelSpec:              `{"chartAppearance": {"points": {"maxSize": 41}}}`,
+			expectedErrorSubstring: "invalid point diameter 41",
+		},
+		{
+			scenario:               "fractional point size",
+			panelKind:              "signoz/ScatterPlotPanel",
+			panelSpec:              `{"chartAppearance": {"points": {"minSize": 4.5}}}`,
+			expectedErrorSubstring: "cannot unmarshal number 4.5",
+		},
+		{
+			scenario:               "min point size above max",
+			panelKind:              "signoz/ScatterPlotPanel",
+			panelSpec:              `{"chartAppearance": {"points": {"minSize": 20, "maxSize": 10}}}`,
+			expectedErrorSubstring: "minSize 20 must not exceed maxSize 10",
+		},
+		{
+			scenario:               "point opacity below the floor",
+			panelKind:              "signoz/ScatterPlotPanel",
+			panelSpec:              `{"chartAppearance": {"points": {"opacity": 0.05}}}`,
+			expectedErrorSubstring: "invalid point opacity 0.05: must be between 0.1 and 1",
+		},
+		{
+			scenario:               "point opacity as a percentage",
+			panelKind:              "signoz/ScatterPlotPanel",
+			panelSpec:              `{"chartAppearance": {"points": {"opacity": 70}}}`,
+			expectedErrorSubstring: "invalid point opacity 70",
+		},
+		{
+			scenario:               "area fill fields on a scatter plot",
+			panelKind:              "signoz/ScatterPlotPanel",
+			panelSpec:              `{"chartAppearance": {"fillOpacity": 0.5}}`,
+			expectedErrorSubstring: "unknown field",
+		},
+		{
+			scenario:               "a single colour key as a string",
+			panelKind:              "signoz/ScatterPlotPanel",
+			panelSpec:              `{"dimensions": {"colorBy": "service.name"}}`,
+			expectedErrorSubstring: "cannot unmarshal string",
+		},
+		{
+			scenario:               "panel-wide unit on a scatter plot panel",
+			panelKind:              "signoz/ScatterPlotPanel",
+			panelSpec:              `{"formatting": {"unit": "ms"}}`,
+			expectedErrorSubstring: "unknown field",
+		},
+		{
+			scenario:               "threshold without a color",
+			panelKind:              "signoz/ScatterPlotPanel",
+			panelSpec:              `{"thresholds": [{"value": 300}]}`,
+			expectedErrorSubstring: "Color' Error:Field validation for 'Color' failed on the 'required' tag",
+		},
+		{
+			scenario:               "scale on a time series panel",
+			panelKind:              "signoz/TimeSeriesPanel",
+			panelSpec:              `{"axes": {"scale": "log"}}`,
+			expectedErrorSubstring: "unknown field",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.scenario, func(t *testing.T) {
+			data := []byte(`{
+				"variables": [],
+				"panels": {
+					"p1": {
+						"kind": "Panel",
+						"spec": {
+							"links": [],
+							"plugin": {"kind": "` + test.panelKind + `", "spec": ` + test.panelSpec + `},
+							"queries": [{"kind": "scalar", "spec": {"plugin": {"kind": "signoz/ClickHouseSQL", "spec": {"name": "A", "query": "SELECT 1"}}}}]
+						}
+					}
+				},
+				"links": [],
+				"layouts": []
+			}`)
+			_, err := unmarshalDashboard(data)
+			require.Error(t, err, "expected the spec to be rejected")
+			assert.Contains(t, err.Error(), test.expectedErrorSubstring, "unexpected error message: %s", err.Error())
+		})
+	}
+}
+
+func TestScatterPlotDimensionsBindToQuery(t *testing.T) {
+	tests := []struct {
+		scenario               string
+		queryPlugin            string
+		dimensions             string
+		expectedErrorSubstring string
+	}{
+		{
+			scenario:    "a single-aggregation query is one value column keyed by its name",
+			queryPlugin: `{"kind": "signoz/BuilderQuery", "spec": {"name": "A", "signal": "traces", "aggregations": [{"expression": "count()"}], "groupBy": [{"name": "service.name"}]}}`,
+			dimensions:  `{"x": "A", "colorBy": ["service.name"]}`,
+		},
+		{
+			scenario:               "a multi-aggregation query keys each value column by name and expression",
+			queryPlugin:            `{"kind": "signoz/BuilderQuery", "spec": {"name": "A", "signal": "traces", "aggregations": [{"expression": "count()"}, {"expression": "p99(duration_nano)"}]}}`,
+			dimensions:             `{"x": "A"}`,
+			expectedErrorSubstring: "dimensions.x: \"A\" is not a value column of the panel's query; value columns: `A.count()`, `A.p99(duration_nano)`",
+		},
+		{
+			scenario:               "a value column of a query the panel does not have",
+			queryPlugin:            `{"kind": "signoz/BuilderQuery", "spec": {"name": "A", "signal": "logs", "aggregations": [{"expression": "count()"}]}}`,
+			dimensions:             `{"y": "B"}`,
+			expectedErrorSubstring: "dimensions.y: \"B\" is not a value column of the panel's query; value columns: `A`",
+		},
+		{
+			scenario:               "a colour key the query does not group by",
+			queryPlugin:            `{"kind": "signoz/BuilderQuery", "spec": {"name": "A", "signal": "traces", "aggregations": [{"expression": "count()"}], "groupBy": [{"name": "service.name"}]}}`,
+			dimensions:             `{"colorBy": ["service.name", "k8s.pod.name"]}`,
+			expectedErrorSubstring: "dimensions.colorBy: \"k8s.pod.name\" is not a group-by key of the panel's query; group-by keys: `service.name`",
+		},
+		{
+			scenario: "a formula is a value column and carries the labels of the disabled queries it reads",
+			queryPlugin: `{"kind": "signoz/CompositeQuery", "spec": {"queries": [
+				{"type": "builder_query", "spec": {"name": "A", "signal": "traces", "disabled": true, "aggregations": [{"expression": "count()"}], "groupBy": [{"name": "service.name"}]}},
+				{"type": "builder_query", "spec": {"name": "B", "signal": "traces", "disabled": true, "aggregations": [{"expression": "countIf(has_error = true)"}], "groupBy": [{"name": "service.name"}]}},
+				{"type": "builder_formula", "spec": {"name": "F1", "expression": "B / A"}}
+			]}}`,
+			dimensions: `{"x": "F1", "colorBy": ["service.name"]}`,
+		},
+		{
+			scenario: "a disabled query yields no value column",
+			queryPlugin: `{"kind": "signoz/CompositeQuery", "spec": {"queries": [
+				{"type": "builder_query", "spec": {"name": "A", "signal": "logs", "disabled": true, "aggregations": [{"expression": "count()"}]}},
+				{"type": "builder_formula", "spec": {"name": "F1", "expression": "A * 2"}}
+			]}}`,
+			dimensions:             `{"sizeBy": "A"}`,
+			expectedErrorSubstring: "dimensions.sizeBy: \"A\" is not a value column of the panel's query; value columns: `F1`",
+		},
+		{
+			scenario:    "clickhouse columns are only known at run time, so the bindings go unchecked",
+			queryPlugin: `{"kind": "signoz/ClickHouseSQL", "spec": {"name": "A", "query": "SELECT 1"}}`,
+			dimensions:  `{"x": "throughput", "colorBy": ["pod"]}`,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.scenario, func(t *testing.T) {
+			data := []byte(`{
+				"variables": [],
+				"panels": {
+					"p1": {
+						"kind": "Panel",
+						"spec": {
+							"links": [],
+							"plugin": {"kind": "signoz/ScatterPlotPanel", "spec": {"dimensions": ` + test.dimensions + `}},
+							"queries": [{"kind": "scalar", "spec": {"plugin": ` + test.queryPlugin + `}}]
+						}
+					}
+				},
+				"links": [],
+				"layouts": []
+			}`)
+			_, err := unmarshalDashboard(data)
+			if test.expectedErrorSubstring == "" {
+				require.NoError(t, err)
+				return
+			}
+			require.Error(t, err, "expected the binding to be rejected")
+			assert.Contains(t, err.Error(), test.expectedErrorSubstring, "unexpected error message: %s", err.Error())
+		})
+	}
+}
+
 func TestNumberPanelDefaults(t *testing.T) {
 	data := []byte(`{
 		"variables": [],
@@ -2228,7 +2541,7 @@ func TestPanelTypeQueryTypeCompatibility(t *testing.T) {
 		switch panelKind {
 		case "signoz/ListPanel":
 			return "raw"
-		case "signoz/TablePanel", "signoz/NumberPanel", "signoz/PieChartPanel", "signoz/HistogramPanel":
+		case "signoz/TablePanel", "signoz/NumberPanel", "signoz/PieChartPanel", "signoz/HistogramPanel", "signoz/ScatterPlotPanel":
 			return "scalar"
 		default:
 			return "time_series"
@@ -2272,14 +2585,17 @@ func TestPanelTypeQueryTypeCompatibility(t *testing.T) {
 		{"List+Builder", mkQuery("signoz/ListPanel", "signoz/BuilderQuery", `{"name":"A","signal":"logs"}`), false},
 		{"TimeSeries+AIBuilder", mkQuery("signoz/TimeSeriesPanel", "signoz/AIBuilderQuery", `{"name":"A","aggregations":[{"expression":"count()"}]}`), false},
 		{"List+AIBuilder", mkQuery("signoz/ListPanel", "signoz/AIBuilderQuery", `{"name":"A"}`), false},
+		{"ScatterPlot+ClickHouse", mkQuery("signoz/ScatterPlotPanel", "signoz/ClickHouseSQL", `{"name":"A","query":"SELECT 1"}`), false},
 		// Top-level: rejected
 		{"Table+PromQL", mkQuery("signoz/TablePanel", "signoz/PromQLQuery", `{"name":"A","query":"up"}`), true},
+		{"ScatterPlot+PromQL", mkQuery("signoz/ScatterPlotPanel", "signoz/PromQLQuery", `{"name":"A","query":"up"}`), true},
 		{"List+ClickHouse", mkQuery("signoz/ListPanel", "signoz/ClickHouseSQL", `{"name":"A","query":"SELECT 1"}`), true},
 		{"List+PromQL", mkQuery("signoz/ListPanel", "signoz/PromQLQuery", `{"name":"A","query":"up"}`), true},
 		{"List+Composite", mkQuery("signoz/ListPanel", "signoz/CompositeQuery", `{"queries":[]}`), true},
 		{"List+Formula", mkQuery("signoz/ListPanel", "signoz/Formula", `{"name":"F1","expression":"A+B"}`), true},
 		// Composite sub-queries
 		{"Table+Composite(promql)", mkComposite("signoz/TablePanel", "promql", `{"name":"A","query":"up"}`), true},
+		{"ScatterPlot+Composite(promql)", mkComposite("signoz/ScatterPlotPanel", "promql", `{"name":"A","query":"up"}`), true},
 		{"Table+Composite(clickhouse)", mkComposite("signoz/TablePanel", "clickhouse_sql", `{"name":"A","query":"SELECT 1"}`), false},
 		{"Table+Composite(builder_ai)", mkComposite("signoz/TablePanel", "builder_ai_query", `{"name":"A","aggregations":[{"expression":"count()"}]}`), false},
 	}

pkg/types/dashboardtypes/perses_plugin_wrappers.go

@@ -28,16 +28,17 @@ type PanelPlugin struct {
 // (and strips the duplicate parent properties) after reflection.
 func (PanelPlugin) PrepareJSONSchema(s *jsonschema.Schema) error {
 	return markDiscriminator(s, "kind", map[string]string{
-		string(PanelKindTimeSeries): schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesTimeSeriesPanelSpec"),
-		string(PanelKindBarChart):   schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesBarChartPanelSpec"),
-		string(PanelKindAreaChart):  schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesAreaChartPanelSpec"),
-		string(PanelKindNumber):     schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesNumberPanelSpec"),
-		string(PanelKindPieChart):   schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesPieChartPanelSpec"),
-		string(PanelKindTable):      schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesTablePanelSpec"),
-		string(PanelKindHistogram):  schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHistogramPanelSpec"),
-		string(PanelKindList):       schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesListPanelSpec"),
-		string(PanelKindText):       schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesTextPanelSpec"),
-		string(PanelKindHeatmap):    schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHeatmapPanelSpec"),
+		string(PanelKindTimeSeries):  schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesTimeSeriesPanelSpec"),
+		string(PanelKindBarChart):    schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesBarChartPanelSpec"),
+		string(PanelKindAreaChart):   schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesAreaChartPanelSpec"),
+		string(PanelKindNumber):      schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesNumberPanelSpec"),
+		string(PanelKindPieChart):    schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesPieChartPanelSpec"),
+		string(PanelKindTable):       schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesTablePanelSpec"),
+		string(PanelKindHistogram):   schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHistogramPanelSpec"),
+		string(PanelKindList):        schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesListPanelSpec"),
+		string(PanelKindText):        schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesTextPanelSpec"),
+		string(PanelKindHeatmap):     schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHeatmapPanelSpec"),
+		string(PanelKindScatterPlot): schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesScatterPlotPanelSpec"),
 	})
 }
 
@@ -71,6 +72,7 @@ func (PanelPlugin) JSONSchemaOneOf() []any {
 		PanelPluginVariant[ListPanelSpec]{Kind: string(PanelKindList)},
 		PanelPluginVariant[TextPanelSpec]{Kind: string(PanelKindText)},
 		PanelPluginVariant[HeatmapPanelSpec]{Kind: string(PanelKindHeatmap)},
+		PanelPluginVariant[ScatterPlotPanelSpec]{Kind: string(PanelKindScatterPlot)},
 	}
 }
 
@@ -234,16 +236,17 @@ func (v VariablePluginVariant[S]) PrepareJSONSchema(s *jsonschema.Schema) error
 
 var (
 	panelPluginSpecs = map[PanelPluginKind]func() any{
-		PanelKindTimeSeries: func() any { return new(TimeSeriesPanelSpec) },
-		PanelKindBarChart:   func() any { return new(BarChartPanelSpec) },
-		PanelKindAreaChart:  func() any { return new(AreaChartPanelSpec) },
-		PanelKindNumber:     func() any { return new(NumberPanelSpec) },
-		PanelKindPieChart:   func() any { return new(PieChartPanelSpec) },
-		PanelKindTable:      func() any { return new(TablePanelSpec) },
-		PanelKindHistogram:  func() any { return new(HistogramPanelSpec) },
-		PanelKindList:       func() any { return new(ListPanelSpec) },
-		PanelKindText:       func() any { return new(TextPanelSpec) },
-		PanelKindHeatmap:    func() any { return new(HeatmapPanelSpec) },
+		PanelKindTimeSeries:  func() any { return new(TimeSeriesPanelSpec) },
+		PanelKindBarChart:    func() any { return new(BarChartPanelSpec) },
+		PanelKindAreaChart:   func() any { return new(AreaChartPanelSpec) },
+		PanelKindNumber:      func() any { return new(NumberPanelSpec) },
+		PanelKindPieChart:    func() any { return new(PieChartPanelSpec) },
+		PanelKindTable:       func() any { return new(TablePanelSpec) },
+		PanelKindHistogram:   func() any { return new(HistogramPanelSpec) },
+		PanelKindList:        func() any { return new(ListPanelSpec) },
+		PanelKindText:        func() any { return new(TextPanelSpec) },
+		PanelKindHeatmap:     func() any { return new(HeatmapPanelSpec) },
+		PanelKindScatterPlot: func() any { return new(ScatterPlotPanelSpec) },
 	}
 	queryPluginSpecs = map[QueryPluginKind]func() any{
 		QueryKindBuilder:       func() any { return new(BuilderQuerySpec) },
@@ -260,16 +263,17 @@ var (
 		VariableKindCustom:  func() any { return new(CustomVariableSpec) },
 	}
 	allowedQueryKinds = map[PanelPluginKind][]QueryPluginKind{
-		PanelKindTimeSeries: {QueryKindBuilder, QueryKindAIBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindPromQL, QueryKindClickHouseSQL},
-		PanelKindBarChart:   {QueryKindBuilder, QueryKindAIBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindPromQL, QueryKindClickHouseSQL},
-		PanelKindAreaChart:  {QueryKindBuilder, QueryKindAIBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindPromQL, QueryKindClickHouseSQL},
-		PanelKindNumber:     {QueryKindBuilder, QueryKindAIBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindPromQL, QueryKindClickHouseSQL},
-		PanelKindHistogram:  {QueryKindBuilder, QueryKindAIBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindPromQL, QueryKindClickHouseSQL},
-		PanelKindPieChart:   {QueryKindBuilder, QueryKindAIBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindClickHouseSQL},
-		PanelKindTable:      {QueryKindBuilder, QueryKindAIBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindClickHouseSQL},
-		PanelKindList:       {QueryKindBuilder, QueryKindAIBuilder},
-		PanelKindText:       {},
-		PanelKindHeatmap:    {QueryKindBuilder, QueryKindComposite, QueryKindFormula, QueryKindPromQL, QueryKindClickHouseSQL},
+		PanelKindTimeSeries:  {QueryKindBuilder, QueryKindAIBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindPromQL, QueryKindClickHouseSQL},
+		PanelKindBarChart:    {QueryKindBuilder, QueryKindAIBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindPromQL, QueryKindClickHouseSQL},
+		PanelKindAreaChart:   {QueryKindBuilder, QueryKindAIBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindPromQL, QueryKindClickHouseSQL},
+		PanelKindNumber:      {QueryKindBuilder, QueryKindAIBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindPromQL, QueryKindClickHouseSQL},
+		PanelKindHistogram:   {QueryKindBuilder, QueryKindAIBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindPromQL, QueryKindClickHouseSQL},
+		PanelKindPieChart:    {QueryKindBuilder, QueryKindAIBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindClickHouseSQL},
+		PanelKindTable:       {QueryKindBuilder, QueryKindAIBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindClickHouseSQL},
+		PanelKindList:        {QueryKindBuilder, QueryKindAIBuilder},
+		PanelKindText:        {},
+		PanelKindHeatmap:     {QueryKindBuilder, QueryKindComposite, QueryKindFormula, QueryKindPromQL, QueryKindClickHouseSQL},
+		PanelKindScatterPlot: {QueryKindBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindClickHouseSQL},
 	}
 )
 

pkg/types/dashboardtypes/perses_public_dashboard.go

@@ -221,7 +221,7 @@ func (d *DashboardV2) GetPanelQuery(startTime, endTime uint64, panelKey string)
 		CompositeQuery: composite,
 		FormatOptions: &qb.FormatOptions{
 			FillGaps:               fillGaps,
-			FormatTableResultForUI: panel.Spec.Plugin.Kind == PanelKindTable,
+			FormatTableResultForUI: panel.Spec.Plugin.Kind == PanelKindTable || panel.Spec.Plugin.Kind == PanelKindScatterPlot,
 		},
 	}, nil
 }

pkg/types/dashboardtypes/perses_public_dashboard_panel_query_test.go

@@ -251,34 +251,38 @@ func TestDashboardV2GetPanelQuery(t *testing.T) {
 		assert.Equal(t, telemetrytypes.SignalTraces, spec.Signal)
 	})
 
-	t.Run("sets FormatTableResultForUI only for table panels", func(t *testing.T) {
-		dashboard := &DashboardV2{
-			Spec: DashboardSpec{
-				Panels: map[string]*Panel{
-					"panel-1": {
-						Spec: PanelSpec{
-							Plugin: PanelPlugin{Kind: PanelKindTable},
-							Queries: []Query{
-								{
-									Kind: qb.RequestTypeScalar,
-									Spec: QuerySpec{
-										Plugin: QueryPlugin{
-											Kind: QueryKindBuilder,
-											Spec: &BuilderQuerySpec{Spec: qb.QueryBuilderQuery[qb.MetricAggregation]{Name: "A"}},
+	t.Run("sets FormatTableResultForUI for panels that join scalar rows", func(t *testing.T) {
+		for _, kind := range []PanelPluginKind{PanelKindTable, PanelKindScatterPlot} {
+			t.Run(string(kind), func(t *testing.T) {
+				dashboard := &DashboardV2{
+					Spec: DashboardSpec{
+						Panels: map[string]*Panel{
+							"panel-1": {
+								Spec: PanelSpec{
+									Plugin: PanelPlugin{Kind: kind},
+									Queries: []Query{
+										{
+											Kind: qb.RequestTypeScalar,
+											Spec: QuerySpec{
+												Plugin: QueryPlugin{
+													Kind: QueryKindBuilder,
+													Spec: &BuilderQuerySpec{Spec: qb.QueryBuilderQuery[qb.MetricAggregation]{Name: "A"}},
+												},
+											},
 										},
 									},
 								},
 							},
 						},
 					},
-				},
-			},
-		}
+				}
 
-		req, err := dashboard.GetPanelQuery(1, 2, "panel-1")
-		require.NoError(t, err)
-		require.NotNil(t, req.FormatOptions)
-		assert.True(t, req.FormatOptions.FormatTableResultForUI)
+				req, err := dashboard.GetPanelQuery(1, 2, "panel-1")
+				require.NoError(t, err)
+				require.NotNil(t, req.FormatOptions)
+				assert.True(t, req.FormatOptions.FormatTableResultForUI)
+			})
+		}
 	})
 
 	t.Run("sets FillGaps from the panel visualization", func(t *testing.T) {

pkg/types/dashboardtypes/perses_signoz_plugins.go

@@ -2,6 +2,8 @@ package dashboardtypes
 
 import (
 	"encoding/json"
+	"maps"
+	"slices"
 	"strconv"
 
 	"github.com/SigNoz/signoz/pkg/errors"
@@ -187,20 +189,21 @@ func (AIBuilderQuerySpec) PrepareJSONSchema(s *jsonschema.Schema) error {
 type PanelPluginKind string
 
 const (
-	PanelKindTimeSeries PanelPluginKind = "signoz/TimeSeriesPanel"
-	PanelKindBarChart   PanelPluginKind = "signoz/BarChartPanel"
-	PanelKindAreaChart  PanelPluginKind = "signoz/AreaChartPanel"
-	PanelKindNumber     PanelPluginKind = "signoz/NumberPanel"
-	PanelKindPieChart   PanelPluginKind = "signoz/PieChartPanel"
-	PanelKindTable      PanelPluginKind = "signoz/TablePanel"
-	PanelKindHistogram  PanelPluginKind = "signoz/HistogramPanel"
-	PanelKindList       PanelPluginKind = "signoz/ListPanel"
-	PanelKindText       PanelPluginKind = "signoz/TextPanel"
-	PanelKindHeatmap    PanelPluginKind = "signoz/HeatmapPanel"
+	PanelKindTimeSeries  PanelPluginKind = "signoz/TimeSeriesPanel"
+	PanelKindBarChart    PanelPluginKind = "signoz/BarChartPanel"
+	PanelKindAreaChart   PanelPluginKind = "signoz/AreaChartPanel"
+	PanelKindNumber      PanelPluginKind = "signoz/NumberPanel"
+	PanelKindPieChart    PanelPluginKind = "signoz/PieChartPanel"
+	PanelKindTable       PanelPluginKind = "signoz/TablePanel"
+	PanelKindHistogram   PanelPluginKind = "signoz/HistogramPanel"
+	PanelKindList        PanelPluginKind = "signoz/ListPanel"
+	PanelKindText        PanelPluginKind = "signoz/TextPanel"
+	PanelKindHeatmap     PanelPluginKind = "signoz/HeatmapPanel"
+	PanelKindScatterPlot PanelPluginKind = "signoz/ScatterPlotPanel"
 )
 
 func (PanelPluginKind) Enum() []any {
-	return []any{PanelKindTimeSeries, PanelKindBarChart, PanelKindAreaChart, PanelKindNumber, PanelKindPieChart, PanelKindTable, PanelKindHistogram, PanelKindList, PanelKindText, PanelKindHeatmap}
+	return []any{PanelKindTimeSeries, PanelKindBarChart, PanelKindAreaChart, PanelKindNumber, PanelKindPieChart, PanelKindTable, PanelKindHistogram, PanelKindList, PanelKindText, PanelKindHeatmap, PanelKindScatterPlot}
 }
 
 func (k PanelPluginKind) rendersWithoutQuery() bool {
@@ -350,6 +353,95 @@ type TextPanelSpec struct {
 	HeaderOptions HeaderOptions    `json:"headerOptions"`
 }
 
+type ScatterPlotPanelSpec struct {
+	Visualization   BasicVisualization         `json:"visualization"`
+	Dimensions      ScatterPlotDimensions      `json:"dimensions"`
+	Formatting      TableFormatting            `json:"formatting"`
+	Axes            ScatterPlotAxes            `json:"axes"`
+	ChartAppearance ScatterPlotChartAppearance `json:"chartAppearance"`
+	Legend          Legend                     `json:"legend"`
+	Thresholds      []ScatterPlotThreshold     `json:"thresholds" validate:"dive"`
+}
+
+func (s *ScatterPlotPanelSpec) validateAgainstQuery(query Query, path string) error {
+	composite, err := query.Spec.Plugin.buildV5CompositeQueryFromPlugin()
+	if err != nil {
+		return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "%s.dimensions: %s", path, err.Error())
+	}
+	keys, known := composite.CollectResultColumnKeys()
+	if !known {
+		return nil
+	}
+	for _, binding := range []struct{ field, key string }{{"x", s.Dimensions.X}, {"y", s.Dimensions.Y}, {"sizeBy", s.Dimensions.SizeBy}} {
+		// Empty is unset: the renderer picks a column or a fixed size.
+		if binding.key == "" {
+			continue
+		}
+		if _, ok := keys.Aggregations[binding.key]; !ok {
+			return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "%s.dimensions.%s: %q is not a value column of the panel's query; value columns: %s", path, binding.field, binding.key, allowedValuesForKind(slices.Sorted(maps.Keys(keys.Aggregations))))
+		}
+	}
+	for _, label := range s.Dimensions.ColorBy {
+		if _, ok := keys.Groups[label]; !ok {
+			return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "%s.dimensions.colorBy: %q is not a group-by key of the panel's query; group-by keys: %s", path, label, allowedValuesForKind(slices.Sorted(maps.Keys(keys.Groups))))
+		}
+	}
+	return nil
+}
+
+// ScatterPlotDimensions binds result columns to what a dot encodes. Value columns
+// use the Table column key, so formatting.columnUnits carries over between the
+// two kinds.
+type ScatterPlotDimensions struct {
+	X       string   `json:"x" description:"Value column key (queryName, or queryName.expression for a multi-aggregation query) plotted on the x axis. Empty uses the first value column."`
+	Y       string   `json:"y" description:"Value column key plotted on the y axis. Empty uses the second value column."`
+	SizeBy  string   `json:"sizeBy" description:"Value column key that scales dot size. Empty draws every dot at the default size."`
+	ColorBy []string `json:"colorBy" description:"Group-by label names (e.g. k8s.namespace.name) whose combined values colour dots and drive the legend. Empty colours by every group-by label."`
+}
+
+type ScatterPlotAxes struct {
+	X ScatterPlotAxis `json:"x"`
+	Y ScatterPlotAxis `json:"y"`
+}
+
+type ScatterPlotAxis struct {
+	SoftMin *float64  `json:"softMin"`
+	SoftMax *float64  `json:"softMax"`
+	Scale   AxisScale `json:"scale"`
+	Label   string    `json:"label" description:"Axis title. Empty draws none."`
+}
+
+type ScatterPlotChartAppearance struct {
+	Points ScatterPlotPoints `json:"points"`
+}
+
+// ScatterPlotPoints keeps the fixed size and the size range side by side so
+// binding or unbinding dimensions.sizeBy restores the other's last setting. Nil
+// fields resolve to the renderer default.
+type ScatterPlotPoints struct {
+	Size    *PointDiameter `json:"size" description:"Diameter of every dot when dimensions.sizeBy is unset."`
+	MinSize *PointDiameter `json:"minSize" description:"Diameter of the smallest dot when dimensions.sizeBy is set."`
+	MaxSize *PointDiameter `json:"maxSize" description:"Diameter of the largest dot when dimensions.sizeBy is set."`
+	Opacity *PointOpacity  `json:"opacity"`
+}
+
+func (p *ScatterPlotPoints) UnmarshalJSON(data []byte) error {
+	type alias ScatterPlotPoints
+	var tmp alias
+	if err := json.Unmarshal(data, &tmp); err != nil {
+		return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid points")
+	}
+	*p = ScatterPlotPoints(tmp)
+	return p.validate()
+}
+
+func (p ScatterPlotPoints) validate() error {
+	if p.MinSize != nil && p.MaxSize != nil && *p.MinSize > *p.MaxSize {
+		return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "invalid points: minSize %d must not exceed maxSize %d", *p.MinSize, *p.MaxSize)
+	}
+	return nil
+}
+
 type TextPresentation struct {
 	TextAlign     TextAlign     `json:"textAlign"`
 	VerticalAlign VerticalAlign `json:"verticalAlign"`
@@ -431,6 +523,11 @@ type ComparisonThreshold struct {
 	Format   ThresholdFormat    `json:"format"`
 }
 
+type ScatterPlotThreshold struct {
+	ThresholdWithLabel
+	Axis ThresholdAxis `json:"axis"`
+}
+
 type TableThreshold struct {
 	ComparisonThreshold
 	ColumnName string `json:"columnName" validate:"required" required:"true"`
@@ -716,6 +813,86 @@ func (ls *LineStyle) UnmarshalJSON(data []byte) error {
 	}
 }
 
+// AxisScale `auto` leaves the choice to the renderer; `symlog` is log-like but
+// defined at and below zero.
+type AxisScale struct{ valuer.String }
+
+var (
+	AxisScaleAuto   = AxisScale{valuer.NewString("auto")} // default
+	AxisScaleLinear = AxisScale{valuer.NewString("linear")}
+	AxisScaleLog    = AxisScale{valuer.NewString("log")}
+	AxisScaleSymlog = AxisScale{valuer.NewString("symlog")}
+)
+
+func (AxisScale) Enum() []any {
+	return []any{AxisScaleAuto, AxisScaleLinear, AxisScaleLog, AxisScaleSymlog}
+}
+
+func (as AxisScale) ValueOrDefault() string {
+	if as.IsZero() {
+		return AxisScaleAuto.StringValue()
+	}
+	return as.StringValue()
+}
+
+func (as AxisScale) MarshalJSON() ([]byte, error) {
+	return json.Marshal(as.ValueOrDefault())
+}
+
+func (as *AxisScale) UnmarshalJSON(data []byte) error {
+	var v string
+	if err := json.Unmarshal(data, &v); err != nil {
+		return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid axis scale: must be a string, one of `auto`, `linear`, `log`, or `symlog`")
+	}
+	val := AxisScale{valuer.NewString(v)}
+	switch val {
+	case AxisScaleAuto, AxisScaleLinear, AxisScaleLog, AxisScaleSymlog:
+		*as = val
+		return nil
+	default:
+		return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "invalid axis scale %q: must be `auto`, `linear`, `log`, or `symlog`", v)
+	}
+}
+
+// ThresholdAxis is the axis a scatter plot threshold line marks: `y` draws it
+// horizontally, `x` vertically.
+type ThresholdAxis struct{ valuer.String }
+
+var (
+	ThresholdAxisX = ThresholdAxis{valuer.NewString("x")}
+	ThresholdAxisY = ThresholdAxis{valuer.NewString("y")} // default
+)
+
+func (ThresholdAxis) Enum() []any {
+	return []any{ThresholdAxisX, ThresholdAxisY}
+}
+
+func (ta ThresholdAxis) ValueOrDefault() string {
+	if ta.IsZero() {
+		return ThresholdAxisY.StringValue()
+	}
+	return ta.StringValue()
+}
+
+func (ta ThresholdAxis) MarshalJSON() ([]byte, error) {
+	return json.Marshal(ta.ValueOrDefault())
+}
+
+func (ta *ThresholdAxis) UnmarshalJSON(data []byte) error {
+	var v string
+	if err := json.Unmarshal(data, &v); err != nil {
+		return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid threshold axis: must be a string, one of `x` or `y`")
+	}
+	val := ThresholdAxis{valuer.NewString(v)}
+	switch val {
+	case ThresholdAxisX, ThresholdAxisY:
+		*ta = val
+		return nil
+	default:
+		return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "invalid threshold axis %q: must be `x` or `y`", v)
+	}
+}
+
 type FillMode struct{ valuer.String }
 
 var (
@@ -854,6 +1031,53 @@ func (o *FillOpacity) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
+const (
+	minPointDiameter = 2
+	maxPointDiameter = 40
+)
+
+// PointDiameter is a scatter dot's diameter in whole CSS pixels.
+type PointDiameter int
+
+func (PointDiameter) PrepareJSONSchema(s *jsonschema.Schema) error {
+	s.WithMinimum(minPointDiameter).WithMaximum(maxPointDiameter)
+	return nil
+}
+
+func (d *PointDiameter) UnmarshalJSON(data []byte) error {
+	var v int
+	if err := json.Unmarshal(data, &v); err != nil {
+		return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid point diameter: must be a whole number between %d and %d", minPointDiameter, maxPointDiameter)
+	}
+	if v < minPointDiameter || v > maxPointDiameter {
+		return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "invalid point diameter %d: must be a whole number between %d and %d", v, minPointDiameter, maxPointDiameter)
+	}
+	*d = PointDiameter(v)
+	return nil
+}
+
+const minPointOpacity = 0.1
+
+// PointOpacity is a scatter dot's fill alpha. The floor keeps a dot visible.
+type PointOpacity float64
+
+func (PointOpacity) PrepareJSONSchema(s *jsonschema.Schema) error {
+	s.WithMinimum(minPointOpacity).WithMaximum(1)
+	return nil
+}
+
+func (o *PointOpacity) UnmarshalJSON(data []byte) error {
+	var v float64
+	if err := json.Unmarshal(data, &v); err != nil {
+		return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid point opacity: must be a number between %v and 1", minPointOpacity)
+	}
+	if v < minPointOpacity || v > 1 {
+		return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "invalid point opacity %v: must be between %v and 1", v, minPointOpacity)
+	}
+	*o = PointOpacity(v)
+	return nil
+}
+
 type SpanGaps struct {
 	FillOnlyBelow bool   `json:"fillOnlyBelow" description:"Controls whether lines connect across null values. When false (default), all gaps are connected. When true, only gaps smaller than fillLessThan are connected."`
 	FillLessThan  string `json:"fillLessThan" description:"The maximum gap size to connect when fillOnlyBelow is true. Gaps larger than this duration are left disconnected."`

pkg/types/dashboardtypes/testdata/perses_scatter_plot_panel.json

@@ -0,0 +1,134 @@
+{
+  "display": {
+    "name": "service health",
+    "description": "throughput against tail latency per pod"
+  },
+  "variables": [],
+  "panels": {
+    "p1": {
+      "kind": "Panel",
+      "spec": {
+        "display": {
+          "name": "throughput vs p99",
+          "description": ""
+        },
+        "plugin": {
+          "kind": "signoz/ScatterPlotPanel",
+          "spec": {
+            "visualization": {
+              "timePreference": "last_1_hr"
+            },
+            "dimensions": {
+              "x": "A.count()",
+              "y": "A.p99(duration_nano)",
+              "sizeBy": "A.countIf(has_error = true)",
+              "colorBy": ["k8s.namespace.name", "k8s.pod.name"]
+            },
+            "formatting": {
+              "columnUnits": {
+                "A.p99(duration_nano)": "ns"
+              },
+              "decimalPrecision": "1"
+            },
+            "axes": {
+              "x": {
+                "softMin": 0,
+                "softMax": null,
+                "scale": "log",
+                "label": "Throughput"
+              },
+              "y": {
+                "softMin": null,
+                "softMax": 1000,
+                "scale": "symlog",
+                "label": "p99 latency"
+              }
+            },
+            "chartAppearance": {
+              "points": {
+                "size": 8,
+                "minSize": 4,
+                "maxSize": 24,
+                "opacity": 0.1
+              }
+            },
+            "legend": {
+              "position": "right",
+              "mode": "list",
+              "customColors": {
+                "checkout": "#1f77b4"
+              }
+            },
+            "thresholds": [
+              {
+                "value": 300,
+                "unit": "ms",
+                "color": "#f00",
+                "label": "p99 SLO",
+                "axis": "y"
+              },
+              {
+                "value": 50,
+                "color": "#0f0",
+                "axis": "x"
+              }
+            ]
+          }
+        },
+        "links": [],
+        "queries": [
+          {
+            "kind": "scalar",
+            "spec": {
+              "plugin": {
+                "kind": "signoz/BuilderQuery",
+                "spec": {
+                  "name": "A",
+                  "signal": "traces",
+                  "aggregations": [
+                    {
+                      "expression": "count()"
+                    },
+                    {
+                      "expression": "p99(duration_nano)"
+                    },
+                    {
+                      "expression": "countIf(has_error = true)"
+                    }
+                  ],
+                  "groupBy": [
+                    {
+                      "name": "k8s.namespace.name"
+                    },
+                    {
+                      "name": "k8s.pod.name"
+                    }
+                  ]
+                }
+              }
+            }
+          }
+        ]
+      }
+    }
+  },
+  "links": [],
+  "layouts": [
+    {
+      "kind": "Grid",
+      "spec": {
+        "items": [
+          {
+            "x": 0,
+            "y": 0,
+            "width": 12,
+            "height": 8,
+            "content": {
+              "$ref": "#/spec/panels/p1"
+            }
+          }
+        ]
+      }
+    }
+  ]
+}

pkg/types/querybuildertypes/querybuildertypesv5/req.go

@@ -266,6 +266,43 @@ type CompositeQuery struct {
 	Queries []QueryEnvelope `json:"queries"`
 }
 
+// ResultColumnKeys is what a scalar query yields per row, keyed the way the
+// UI keys columns: an aggregation column is the query name, or
+// queryName.expression when the query has several aggregations; a group
+// column is a group-by key name.
+type ResultColumnKeys struct {
+	Aggregations map[string]struct{}
+	Groups       map[string]struct{}
+}
+
+// CollectResultColumnKeys reports false when a query's columns are only known
+// at run time, as for ClickHouse SQL and PromQL. A disabled query yields no
+// aggregation column but keeps its group columns, since a formula over it
+// still carries them.
+func (c *CompositeQuery) CollectResultColumnKeys() (ResultColumnKeys, bool) {
+	keys := ResultColumnKeys{Aggregations: map[string]struct{}{}, Groups: map[string]struct{}{}}
+	for _, envelope := range c.Queries {
+		if envelope.GetType() == QueryTypeClickHouseSQL || envelope.GetType() == QueryTypePromQL {
+			return ResultColumnKeys{}, false
+		}
+		for _, key := range envelope.GetGroupBy() {
+			keys.Groups[key.Name] = struct{}{}
+		}
+		if envelope.IsDisabled() {
+			continue
+		}
+		expressions := envelope.GetAggregationExpressions()
+		if len(expressions) <= 1 {
+			keys.Aggregations[envelope.GetQueryName()] = struct{}{}
+			continue
+		}
+		for _, expression := range expressions {
+			keys.Aggregations[envelope.GetQueryName()+"."+expression] = struct{}{}
+		}
+	}
+	return keys, true
+}
+
 // PrepareJSONSchema adds description to the CompositeQuery schema.
 func (c *CompositeQuery) PrepareJSONSchema(schema *jsonschema.Schema) error {
 	schema.WithDescription("Composite query containing one or more query envelopes. Each query envelope specifies its type and corresponding spec.")

pkg/types/querybuildertypes/querybuildertypesv5/req_getters.go

@@ -225,6 +225,34 @@ func (q *QueryEnvelope) GetGroupBy() []GroupByKey {
 	return nil
 }
 
+// GetAggregationExpressions returns one entry per aggregation. A metric
+// aggregation has no expression, so it contributes an empty entry.
+func (q *QueryEnvelope) GetAggregationExpressions() []string {
+	switch spec := q.Spec.(type) {
+	case QueryBuilderTraceOperator:
+		expressions := make([]string, len(spec.Aggregations))
+		for i, aggregation := range spec.Aggregations {
+			expressions[i] = aggregation.Expression
+		}
+		return expressions
+	case QueryBuilderQuery[TraceAggregation]:
+		expressions := make([]string, len(spec.Aggregations))
+		for i, aggregation := range spec.Aggregations {
+			expressions[i] = aggregation.Expression
+		}
+		return expressions
+	case QueryBuilderQuery[LogAggregation]:
+		expressions := make([]string, len(spec.Aggregations))
+		for i, aggregation := range spec.Aggregations {
+			expressions[i] = aggregation.Expression
+		}
+		return expressions
+	case QueryBuilderQuery[MetricAggregation]:
+		return make([]string, len(spec.Aggregations))
+	}
+	return nil
+}
+
 // GetFilter returns the filter.
 func (q *QueryEnvelope) GetFilter() *Filter {
 	switch spec := q.Spec.(type) {

pkg/types/querybuildertypes/querybuildertypesv5/req_test.go

@@ -1903,3 +1903,63 @@ func TestQueryRangeRequest_StepIntervalForQuery(t *testing.T) {
 		})
 	}
 }
+
+func TestCompositeQueryCollectResultColumnKeys(t *testing.T) {
+	testCases := []struct {
+		scenario      string
+		composite     CompositeQuery
+		expectedKeys  ResultColumnKeys
+		expectedKnown bool
+	}{
+		{
+			scenario: "a single aggregation is keyed by the query name",
+			composite: CompositeQuery{Queries: []QueryEnvelope{{Type: QueryTypeBuilder, Spec: QueryBuilderQuery[TraceAggregation]{
+				Name:         "A",
+				Aggregations: []TraceAggregation{{Expression: "count()"}},
+				GroupBy:      []GroupByKey{{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}}},
+			}}}},
+			expectedKeys:  ResultColumnKeys{Aggregations: map[string]struct{}{"A": {}}, Groups: map[string]struct{}{"service.name": {}}},
+			expectedKnown: true,
+		},
+		{
+			scenario: "several aggregations are keyed by query name and expression",
+			composite: CompositeQuery{Queries: []QueryEnvelope{{Type: QueryTypeBuilder, Spec: QueryBuilderQuery[LogAggregation]{
+				Name:         "A",
+				Aggregations: []LogAggregation{{Expression: "count()"}, {Expression: "p99(duration_nano)"}},
+			}}}},
+			expectedKeys:  ResultColumnKeys{Aggregations: map[string]struct{}{"A.count()": {}, "A.p99(duration_nano)": {}}, Groups: map[string]struct{}{}},
+			expectedKnown: true,
+		},
+		{
+			scenario: "a disabled query yields no aggregation but keeps its groups for the formula over it",
+			composite: CompositeQuery{Queries: []QueryEnvelope{
+				{Type: QueryTypeBuilder, Spec: QueryBuilderQuery[MetricAggregation]{
+					Name:         "A",
+					Disabled:     true,
+					Aggregations: []MetricAggregation{{MetricName: "http.server.request.count"}},
+					GroupBy:      []GroupByKey{{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}}},
+				}},
+				{Type: QueryTypeFormula, Spec: QueryBuilderFormula{Name: "F1", Expression: "A * 2"}},
+			}},
+			expectedKeys:  ResultColumnKeys{Aggregations: map[string]struct{}{"F1": {}}, Groups: map[string]struct{}{"service.name": {}}},
+			expectedKnown: true,
+		},
+		{
+			scenario: "clickhouse columns are only known at run time",
+			composite: CompositeQuery{Queries: []QueryEnvelope{{Type: QueryTypeClickHouseSQL, Spec: ClickHouseQuery{
+				Name:  "A",
+				Query: "SELECT 1",
+			}}}},
+			expectedKeys:  ResultColumnKeys{},
+			expectedKnown: false,
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.scenario, func(t *testing.T) {
+			keys, known := testCase.composite.CollectResultColumnKeys()
+			assert.Equal(t, testCase.expectedKnown, known)
+			assert.Equal(t, testCase.expectedKeys, keys)
+		})
+	}
+}

pkg/types/spantypes/thread.go

@@ -178,8 +178,9 @@ func newThreadSpan(traceID string, storable *StorableSpan) *ThreadSpan {
 	maps.Copy(resources, storable.ResourcesString)
 	timeUnixNano := uint64(storable.StartTime.UnixNano())
 	attributes := threadAttributes(storable)
-	// Pre-rollout spans have an empty JSON document and format to nothing.
-	formatted := aiobservabilitytypes.FormatMessages(FlatJSONAttributes(storable.AttributesJSON))
+	// Spans from before the JSON rollout have no document and are not decoded: the legacy maps
+	// hold an object split into one key per field.
+	formatted := aiobservabilitytypes.FormatMessages(storable.AttributesJSON)
 	return &ThreadSpan{
 		SpanID:            storable.SpanID,
 		TraceID:           traceID,
@@ -202,12 +203,13 @@ func newThreadSpan(traceID string, storable *StorableSpan) *ThreadSpan {
 	}
 }
 
-// threadAttributes returns the span's flat dotted attributes, as the querier does for list
+// threadAttributes flattens the JSON column into dotted keys, as the querier does for list
 // responses, and falls back to the legacy maps for spans written before the JSON rollout.
 func threadAttributes(storable *StorableSpan) map[string]any {
-	attributes := FlatJSONAttributes(storable.AttributesJSON)
-	if len(attributes) == 0 {
+	if len(storable.AttributesJSON) == 0 {
 		return storable.Attributes()
 	}
+	attributes := make(map[string]any, len(storable.AttributesJSON))
+	storable.AttributesJSON.FlattenInto("", attributes)
 	return attributes
 }

pkg/types/spantypes/thread_test.go

@@ -3,21 +3,11 @@ package spantypes
 import (
 	"testing"
 
-	"github.com/ClickHouse/clickhouse-go/v2/lib/chcol"
-	"github.com/SigNoz/signoz/pkg/types/aiobservabilitytypes"
+	"github.com/SigNoz/signoz/pkg/types/telemetrystoretypes"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// jsonAttrs builds a native JSON column from dotted leaf paths, as ClickHouse stores them.
-func jsonAttrs(paths map[string]any) chcol.JSON {
-	j := chcol.NewJSON()
-	for path, value := range paths {
-		j.SetValueAtPath(path, value)
-	}
-	return *j
-}
-
 func TestNewThreadQuery_Cursor(t *testing.T) {
 	testCases := []struct {
 		name    string
@@ -58,7 +48,7 @@ func TestThreadAttributes(t *testing.T) {
 		{
 			name: "JSONColumnPresent_LegacyMapsIgnored",
 			span: StorableSpan{
-				AttributesJSON:   jsonAttrs(map[string]any{"gen_ai.request.model": "json"}),
+				AttributesJSON:   telemetrystoretypes.JSONValue{"gen_ai": map[string]any{"request": map[string]any{"model": "json"}}},
 				AttributesString: map[string]string{"gen_ai.request.model": "map", "http.method": "GET"},
 			},
 			wantAttrs: map[string]any{"gen_ai.request.model": "json"},
@@ -76,22 +66,6 @@ func TestThreadAttributes(t *testing.T) {
 				"gen_ai.stream":             true,
 			},
 		},
-		{
-			// ClickHouse materializes a row's absent paths as null from the part's path schema.
-			name: "JSONColumn_DropsNullPaths",
-			span: StorableSpan{
-				AttributesJSON: jsonAttrs(map[string]any{"gen_ai.request.model": "gpt-4o", "gen_ai.output.messages": chcol.Variant{}}),
-			},
-			wantAttrs: map[string]any{"gen_ai.request.model": "gpt-4o"},
-		},
-		{
-			name: "JSONColumn_OnlyNullPaths_FallsBackToLegacyMaps",
-			span: StorableSpan{
-				AttributesJSON:   jsonAttrs(map[string]any{"gen_ai.output.messages": chcol.Variant{}}),
-				AttributesString: map[string]string{"gen_ai.input.messages": "legacy"},
-			},
-			wantAttrs: map[string]any{"gen_ai.input.messages": "legacy"},
-		},
 	}
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
@@ -99,20 +73,3 @@ func TestThreadAttributes(t *testing.T) {
 		})
 	}
 }
-
-func TestNewThreadSpan_FormatsJSONMessages(t *testing.T) {
-	storable := StorableSpan{
-		SpanID: "f1fa1bc863e94dd0",
-		AttributesJSON: jsonAttrs(map[string]any{
-			"gen_ai.input.messages":  []any{map[string]any{"role": "user", "content": "hi"}},
-			"gen_ai.output.messages": []any{map[string]any{"role": "assistant", "content": "hello"}},
-		}),
-	}
-
-	span := newThreadSpan("trace", &storable)
-
-	require.Len(t, span.FormattedInput, 1)
-	assert.Equal(t, aiobservabilitytypes.RoleUser, span.FormattedInput[0].Role)
-	require.Len(t, span.FormattedOutput, 1)
-	assert.Equal(t, aiobservabilitytypes.RoleAssistant, span.FormattedOutput[0].Role)
-}

pkg/types/spantypes/waterfall_span.go

@@ -7,10 +7,9 @@ import (
 	"sort"
 	"time"
 
-	"github.com/ClickHouse/clickhouse-go/v2/lib/chcol"
-
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/types/aiobservabilitytypes"
+	"github.com/SigNoz/signoz/pkg/types/telemetrystoretypes"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 )
 
@@ -102,36 +101,36 @@ type WaterfallSpan struct {
 
 // StorableSpan is the ClickHouse scan struct for the v3 waterfall query.
 type StorableSpan struct {
-	StartTime          time.Time          `ch:"timestamp"`
-	DurationNano       uint64             `ch:"duration_nano"`
-	SpanID             string             `ch:"span_id"`
-	HasError           bool               `ch:"has_error"`
-	Kind               int8               `ch:"kind"`
-	ServiceName        string             `ch:"resource_string_service$$name"`
-	Name               string             `ch:"name"`
-	AttributesString   map[string]string  `ch:"attributes_string"`
-	AttributesNumber   map[string]float64 `ch:"attributes_number"`
-	AttributesBool     map[string]bool    `ch:"attributes_bool"`
-	AttributesJSON     chcol.JSON         `ch:"attributes"`
-	ResourcesString    map[string]string  `ch:"resources_string"`
-	Events             []string           `ch:"events"`
-	StatusMessage      string             `ch:"status_message"`
-	StatusCodeString   string             `ch:"status_code_string"`
-	SpanKind           string             `ch:"kind_string"`
-	ParentSpanID       string             `ch:"parent_span_id"`
-	Flags              uint32             `ch:"flags"`
-	IsRemote           string             `ch:"is_remote"`
-	TraceState         string             `ch:"trace_state"`
-	StatusCode         int16              `ch:"status_code"`
-	DBName             string             `ch:"db_name"`
-	DBOperation        string             `ch:"db_operation"`
-	HTTPMethod         string             `ch:"http_method"`
-	HTTPURL            string             `ch:"http_url"`
-	HTTPHost           string             `ch:"http_host"`
-	ExternalHTTPMethod string             `ch:"external_http_method"`
-	ExternalHTTPURL    string             `ch:"external_http_url"`
-	ResponseStatusCode string             `ch:"response_status_code"`
-	References         string             `ch:"references"`
+	StartTime          time.Time                     `ch:"timestamp"`
+	DurationNano       uint64                        `ch:"duration_nano"`
+	SpanID             string                        `ch:"span_id"`
+	HasError           bool                          `ch:"has_error"`
+	Kind               int8                          `ch:"kind"`
+	ServiceName        string                        `ch:"resource_string_service$$name"`
+	Name               string                        `ch:"name"`
+	AttributesString   map[string]string             `ch:"attributes_string"`
+	AttributesNumber   map[string]float64            `ch:"attributes_number"`
+	AttributesBool     map[string]bool               `ch:"attributes_bool"`
+	AttributesJSON     telemetrystoretypes.JSONValue `ch:"attributes"`
+	ResourcesString    map[string]string             `ch:"resources_string"`
+	Events             []string                      `ch:"events"`
+	StatusMessage      string                        `ch:"status_message"`
+	StatusCodeString   string                        `ch:"status_code_string"`
+	SpanKind           string                        `ch:"kind_string"`
+	ParentSpanID       string                        `ch:"parent_span_id"`
+	Flags              uint32                        `ch:"flags"`
+	IsRemote           string                        `ch:"is_remote"`
+	TraceState         string                        `ch:"trace_state"`
+	StatusCode         int16                         `ch:"status_code"`
+	DBName             string                        `ch:"db_name"`
+	DBOperation        string                        `ch:"db_operation"`
+	HTTPMethod         string                        `ch:"http_method"`
+	HTTPURL            string                        `ch:"http_url"`
+	HTTPHost           string                        `ch:"http_host"`
+	ExternalHTTPMethod string                        `ch:"external_http_method"`
+	ExternalHTTPURL    string                        `ch:"external_http_url"`
+	ResponseStatusCode string                        `ch:"response_status_code"`
+	References         string                        `ch:"references"`
 }
 
 // MinimalSpan with only the fields needed to build the parent-child tree.
@@ -301,23 +300,6 @@ func (item *StorableSpan) Attributes() map[string]any {
 	return attributes
 }
 
-// FlatJSONAttributes returns the JSON attributes column keyed by full dotted path, dropping the null
-// paths ClickHouse materializes for a row from its data part's path schema.
-func FlatJSONAttributes(attributes chcol.JSON) map[string]any {
-	paths := attributes.ValuesByPath()
-	out := make(map[string]any, len(paths))
-	for path, value := range paths {
-		if value == nil {
-			continue
-		}
-		if variant, ok := value.(chcol.Variant); ok && variant.Nil() {
-			continue
-		}
-		out[path] = value
-	}
-	return out
-}
-
 func (item *StorableSpan) UnmarshalledEvents() []Event {
 	events := make([]Event, 0, len(item.Events))
 	for _, eventStr := range item.Events {
@@ -342,7 +324,7 @@ func (item *StorableSpan) ToWaterfallSpan(traceID string) *WaterfallSpan {
 	resources := make(map[string]string)
 	maps.Copy(resources, item.ResourcesString)
 
-	formatted := aiobservabilitytypes.FormatMessages(FlatJSONAttributes(item.AttributesJSON))
+	formatted := aiobservabilitytypes.FormatMessages(item.AttributesJSON)
 	return &WaterfallSpan{
 		Attributes:         item.Attributes(),
 		DBName:             item.DBName,

pkg/types/telemetrystoretypes/json.go

@@ -0,0 +1,55 @@
+package telemetrystoretypes
+
+import (
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/bytedance/sonic"
+)
+
+var ErrCodeUnmarshalJSONColumn = errors.MustNewCode("fail_unmarshal_json_column")
+
+// JSONValue is the scan target for a ClickHouse JSON column: the connection sets
+// output_format_native_write_json_as_string, so the column arrives as a raw document rather than
+// the chcol.JSON the driver reports as its scan type.
+type JSONValue map[string]any
+
+// Scan decodes into a fresh map every time: a scan target is reused across rows, and unmarshalling
+// into the map already there would both keep its keys and hand every row the same map.
+func (v *JSONValue) Scan(src any) error {
+	var raw []byte
+	switch value := src.(type) {
+	case nil:
+		*v = nil
+		return nil
+	case string:
+		raw = []byte(value)
+	case []byte:
+		raw = value
+	default:
+		return errors.NewInternalf(ErrCodeUnmarshalJSONColumn, "cannot decode %T as a JSON column", src)
+	}
+
+	decoded := JSONValue{}
+	if err := sonic.Unmarshal(raw, &decoded); err != nil {
+		return errors.WrapInternalf(err, ErrCodeUnmarshalJSONColumn, "failed to unmarshal JSON column")
+	}
+	*v = decoded
+	return nil
+}
+
+// FlattenInto writes v into out under dotted keys, overwriting existing keys.
+func (v JSONValue) FlattenInto(prefix string, out map[string]any) {
+	for k, value := range v {
+		key := k
+		if prefix != "" {
+			key = prefix + "." + k
+		}
+		switch child := value.(type) {
+		case map[string]any:
+			JSONValue(child).FlattenInto(key, out)
+		case JSONValue:
+			child.FlattenInto(key, out)
+		default:
+			out[key] = value
+		}
+	}
+}

tests/integration/tests/querier_json_body/09_scalar_object_body.py

@@ -1,52 +0,0 @@
-import json
-from collections.abc import Callable
-from datetime import UTC, datetime, timedelta
-from http import HTTPStatus
-
-from fixtures import types
-from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
-from fixtures.logs import Logs
-from fixtures.querier import RequestType, build_raw_query, get_rows, make_query_request
-
-SERVICE = "scalar-object-body"
-
-
-def test_scalar_and_object_body_key_collapses(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_logs: Callable[[list[Logs]], None],
-    export_json_types: Callable[[list[Logs]], None],
-) -> None:
-    """A log body key stored as both a scalar and an object (db.function and db.function.arg) keeps
-    only one: the body is returned nested, not flattened to dotted keys, so it cannot hold both."""
-    now = datetime.now(tz=UTC)
-    logs = [
-        Logs(
-            timestamp=now - timedelta(seconds=1),
-            resources={"service.name": SERVICE},
-            body_v2=json.dumps({"db.function": "refresh", "db.function.arg": 2}),
-            body_promoted="",
-        ),
-    ]
-    export_json_types(logs)
-    insert_logs(logs)
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-
-    response = make_query_request(
-        signoz,
-        token,
-        start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
-        end_ms=int(now.timestamp() * 1000),
-        request_type=RequestType.RAW,
-        queries=[build_raw_query("A", "logs", limit=10, filter_expression=f"service.name = '{SERVICE}'")],
-    )
-    assert response.status_code == HTTPStatus.OK, response.text
-
-    rows = get_rows(response)
-    assert len(rows) == 1
-    body = rows[0]["data"]["body"]
-    # The scalar survives as a string; the sibling object path db.function.arg is not preserved
-    # under it, so db.function is a plain value rather than an object.
-    assert body["db"]["function"] == "refresh"
-    assert isinstance(body["db"]["function"], str)

tests/integration/tests/queriertraces/14_scalar_object_attribute.py

@@ -1,74 +0,0 @@
-from collections.abc import Callable
-from datetime import UTC, datetime, timedelta
-from http import HTTPStatus
-
-from fixtures import types
-from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
-from fixtures.querier import (
-    BuilderQuery,
-    OrderBy,
-    RequestType,
-    TelemetryFieldKey,
-    get_rows,
-    make_query_request,
-)
-from fixtures.traces import TraceIdGenerator, Traces
-
-
-def test_traces_scalar_and_object_key_both_survive(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_traces: Callable[[list[Traces]], None],
-    seed_attribute_evolution: Callable[[str, datetime], None],
-) -> None:
-    """A span attribute stored as both a scalar and an object prefix (db.function and
-    db.function.arg) keeps both dotted keys through the native JSON read. json_only writes only the
-    JSON column, so no legacy attribute map backfills the keys."""
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-
-    evolution_time = datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=30)
-    seed_attribute_evolution("traces", evolution_time)
-
-    service = "scalar-object-service"
-    span_time = evolution_time + timedelta(minutes=5)
-    insert_traces(
-        [
-            Traces(
-                timestamp=span_time,
-                trace_id=TraceIdGenerator.trace_id(),
-                span_id=TraceIdGenerator.span_id(),
-                name="scalar and object",
-                resources={"service.name": service},
-                attributes={"db.function": "refresh", "db.function.arg": 2},
-                attribute_write_mode="json_only",
-            ),
-        ]
-    )
-
-    response = make_query_request(
-        signoz,
-        token,
-        start_ms=int((span_time - timedelta(minutes=1)).timestamp() * 1000),
-        end_ms=int((span_time + timedelta(minutes=1)).timestamp() * 1000),
-        request_type=RequestType.RAW,
-        queries=[
-            BuilderQuery(
-                signal="traces",
-                name="A",
-                limit=10,
-                filter_expression=f"resource.service.name = '{service}'",
-                order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
-            ).to_dict()
-        ],
-    )
-    assert response.status_code == HTTPStatus.OK, response.text
-
-    rows = get_rows(response)
-    assert len(rows) == 1
-    attributes = rows[0]["data"]["attributes"]
-    assert attributes["db.function"] == "refresh"
-    assert isinstance(attributes["db.function"], str)
-    assert attributes["db.function.arg"] == 2
-    # the native read returns flat dotted keys, so no Go-side flattening is needed: db is not nested
-    assert "db" not in attributes

tests/integration/tests/tracedetail/02_thread.py

@@ -21,25 +21,17 @@ def test_thread_returns_gen_ai_spans_in_order(
     seed_attribute_evolution("traces", ATTRIBUTE_JSON_ROLLOUT_TIME)
     now = datetime.now(tz=UTC).replace(microsecond=0)
     trace_id = TraceIdGenerator.trace_id()
-    root_id, first_llm_id, tool_id, agent_id, second_llm_id, legacy_llm_id, third_llm_id, flattened_llm_id, responses_llm_id, langchain_llm_id, openrouter_llm_id, vercel_llm_id, anthropic_llm_id, gemini_llm_id = (TraceIdGenerator.span_id() for _ in range(14))
+    root_id, first_llm_id, tool_id, agent_id, second_llm_id, third_llm_id = (TraceIdGenerator.span_id() for _ in range(6))
     resources = {"service.name": "tracedetail-thread"}
     first_input = json.dumps([{"role": "user", "parts": [{"type": "text", "content": "weather in Bangalore?"}]}])
-    first_output = json.dumps([{"role": "assistant", "parts": [{"type": "tool_call", "id": "call_1", "name": "get_weather", "arguments": {"city": "Bangalore"}}]}])
-    first_instructions = json.dumps([{"type": "text", "content": "Answer with the tool."}])
+    first_output = json.dumps([{"role": "assistant", "parts": [{"type": "tool_call", "id": "call_1", "name": "get_weather", "arguments": {"city": "Bangalore"}}], "finish_reason": "tool_call"}])
     second_input = json.dumps([{"role": "tool", "content": "sunny", "tool_call_id": "call_1"}])
 
     insert_traces(
         [
             Traces(timestamp=now - timedelta(seconds=10), duration=timedelta(seconds=9), trace_id=trace_id, span_id=root_id, name="POST /chat", kind=TracesKind.SPAN_KIND_SERVER, resources=resources, attribute_write_mode="json_only"),
             Traces(
-                timestamp=now - timedelta(seconds=8),
-                trace_id=trace_id,
-                span_id=first_llm_id,
-                parent_span_id=root_id,
-                name="chat gpt-4o",
-                resources=resources,
-                attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.system_instructions": first_instructions, "gen_ai.input.messages": first_input, "gen_ai.output.messages": first_output, "gen_ai.response.finish_reasons": json.dumps(["tool_calls"])},
-                attribute_write_mode="json_only",
+                timestamp=now - timedelta(seconds=8), trace_id=trace_id, span_id=first_llm_id, parent_span_id=root_id, name="chat gpt-4o", resources=resources, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.input.messages": first_input, "gen_ai.output.messages": first_output}, attribute_write_mode="json_only"
             ),
             Traces(
                 timestamp=now - timedelta(seconds=6),
@@ -53,122 +45,7 @@ def test_thread_returns_gen_ai_spans_in_order(
             ),
             Traces(timestamp=now - timedelta(seconds=5), trace_id=trace_id, span_id=agent_id, parent_span_id=root_id, name="invoke_agent planner", resources=resources, attributes={"gen_ai.agent.name": "planner"}, attribute_write_mode="json_only"),
             Traces(timestamp=now - timedelta(seconds=4), trace_id=trace_id, span_id=second_llm_id, parent_span_id=root_id, name="chat gpt-4o", resources=resources, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.input.messages": second_input}, attribute_write_mode="json_only"),
-            Traces(
-                timestamp=now - timedelta(seconds=3),
-                trace_id=trace_id,
-                span_id=legacy_llm_id,
-                parent_span_id=root_id,
-                name="openai.chat",
-                resources=resources,
-                attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.prompt.0.role": "user", "gen_ai.prompt.0.content": "and tomorrow?", "gen_ai.completion.0.role": "assistant", "gen_ai.completion.0.content": "Also sunny.", "gen_ai.completion.0.finish_reason": "stop"},
-                attribute_write_mode="json_only",
-            ),
             Traces(timestamp=now - timedelta(seconds=2), trace_id=trace_id, span_id=third_llm_id, parent_span_id=root_id, name="chat gpt-4o", resources=resources, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.output.messages": "It is sunny in Bangalore."}, attribute_write_mode="json_only"),
-            Traces(
-                timestamp=now - timedelta(seconds=1),
-                trace_id=trace_id,
-                span_id=flattened_llm_id,
-                parent_span_id=root_id,
-                name="ChatCompletion",
-                resources=resources,
-                attributes={
-                    "gen_ai.request.model": "gpt-4o",
-                    "llm.input_messages.0.message.role": "user",
-                    "llm.input_messages.0.message.content": "thanks",
-                    "llm.output_messages.0.message.role": "assistant",
-                    "llm.output_messages.0.message.content": "You're welcome.",
-                    "llm.finish_reason": "stop",
-                },
-                attribute_write_mode="json_only",
-            ),
-            Traces(
-                timestamp=now,
-                trace_id=trace_id,
-                span_id=responses_llm_id,
-                parent_span_id=root_id,
-                name="responses gpt-4o",
-                resources=resources,
-                attributes={
-                    "gen_ai.request.model": "gpt-4o",
-                    "gen_ai.input.messages": json.dumps({"model": "gpt-4o", "instructions": "Be brief.", "input": [{"role": "user", "content": "bye"}]}),
-                    "gen_ai.output.messages": json.dumps({"object": "response", "status": "completed", "output": [{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Bye!"}]}]}),
-                },
-                attribute_write_mode="json_only",
-            ),
-            Traces(
-                timestamp=now + timedelta(seconds=1),
-                trace_id=trace_id,
-                span_id=langchain_llm_id,
-                parent_span_id=root_id,
-                name="ChatOpenAI",
-                resources=resources,
-                attributes={
-                    "gen_ai.request.model": "gpt-4o",
-                    "gen_ai.input.messages": json.dumps({"messages": [[{"lc": 1, "type": "constructor", "id": ["langchain", "schema", "messages", "HumanMessage"], "kwargs": {"content": "see you", "type": "human"}}]]}),
-                    "gen_ai.output.messages": json.dumps(
-                        {"generations": [[{"text": "See you!", "generation_info": {"finish_reason": "stop"}, "type": "ChatGeneration", "message": {"lc": 1, "type": "constructor", "id": ["langchain", "schema", "messages", "AIMessage"], "kwargs": {"content": "See you!", "type": "ai"}}}]]}
-                    ),
-                },
-                attribute_write_mode="json_only",
-            ),
-            Traces(
-                timestamp=now + timedelta(seconds=2),
-                trace_id=trace_id,
-                span_id=openrouter_llm_id,
-                parent_span_id=root_id,
-                name="LLM Generation",
-                resources=resources,
-                attributes={
-                    "gen_ai.request.model": "openai/gpt-4o-mini",
-                    "gen_ai.input.messages": json.dumps({"messages": [{"role": "user", "content": "one word"}]}),
-                    "gen_ai.output.messages": json.dumps({"completion": "Yes.", "reasoning": None, "rawRequest": {"model": "openai/gpt-4o-mini"}}),
-                },
-                attribute_write_mode="json_only",
-            ),
-            Traces(
-                timestamp=now + timedelta(seconds=3),
-                trace_id=trace_id,
-                span_id=vercel_llm_id,
-                parent_span_id=root_id,
-                name="ai.generateText.doGenerate",
-                resources=resources,
-                attributes={
-                    "gen_ai.request.model": "gpt-4o-mini",
-                    "ai.operationId": "ai.generateText.doGenerate",
-                    "ai.response.finishReason": "stop",
-                    "gen_ai.input.messages": json.dumps([{"role": "user", "content": [{"type": "text", "text": "last one"}]}]),
-                    "gen_ai.output.messages": "Done.",
-                },
-                attribute_write_mode="json_only",
-            ),
-            Traces(
-                timestamp=now + timedelta(seconds=4),
-                trace_id=trace_id,
-                span_id=anthropic_llm_id,
-                parent_span_id=root_id,
-                name="messages claude-sonnet-4-5",
-                resources=resources,
-                attributes={
-                    "gen_ai.request.model": "claude-sonnet-4-5",
-                    "gen_ai.input.messages": json.dumps({"system": "Be terse.", "messages": [{"role": "user", "content": "ping"}]}),
-                    "gen_ai.output.messages": json.dumps({"type": "message", "role": "assistant", "content": [{"type": "text", "text": "pong"}], "stop_reason": "end_turn"}),
-                },
-                attribute_write_mode="json_only",
-            ),
-            Traces(
-                timestamp=now + timedelta(seconds=5),
-                trace_id=trace_id,
-                span_id=gemini_llm_id,
-                parent_span_id=root_id,
-                name="generateContent gemini-2.5-flash",
-                resources=resources,
-                attributes={
-                    "gen_ai.request.model": "gemini-2.5-flash",
-                    "gen_ai.input.messages": json.dumps({"contents": [{"role": "user", "parts": [{"text": "ping"}]}]}),
-                    "gen_ai.output.messages": json.dumps({"candidates": [{"content": {"role": "model", "parts": [{"text": "pong"}]}, "finishReason": "STOP"}]}),
-                },
-                attribute_write_mode="json_only",
-            ),
         ]
     )
 
@@ -177,10 +54,10 @@ def test_thread_returns_gen_ai_spans_in_order(
     assert response.status_code == HTTPStatus.OK, response.text
 
     thread = response.json()["data"]
-    assert [span["span_id"] for span in thread["spans"]] == [first_llm_id, tool_id, agent_id, second_llm_id, legacy_llm_id, third_llm_id, flattened_llm_id, responses_llm_id, langchain_llm_id, openrouter_llm_id, vercel_llm_id, anthropic_llm_id, gemini_llm_id]
+    assert [span["span_id"] for span in thread["spans"]] == [first_llm_id, tool_id, agent_id, second_llm_id, third_llm_id]
     assert "nextCursor" not in thread
 
-    first, tool, agent, input_only, legacy, output_only, flattened, responses, langchain, openrouter, vercel, anthropic, gemini = thread["spans"]
+    first, tool, agent, input_only, output_only = thread["spans"]
     assert agent["attributes"]["gen_ai.agent.name"] == "planner"
     assert agent["formattedInput"] == []
     assert agent["formattedOutput"] == []
@@ -196,7 +73,7 @@ def test_thread_returns_gen_ai_spans_in_order(
     assert output_only["attributes"]["gen_ai.output.messages"] == "It is sunny in Bangalore."
 
     tool_call = {"type": "tool_call", "name": "get_weather", "id": "call_1", "arguments": {"city": "Bangalore"}}
-    assert first["formattedInput"] == [{"role": "system", "parts": [{"type": "text", "content": "Answer with the tool."}]}, {"role": "user", "parts": [{"type": "text", "content": "weather in Bangalore?"}]}]
+    assert first["formattedInput"] == [{"role": "user", "parts": [{"type": "text", "content": "weather in Bangalore?"}]}]
     assert first["formattedOutput"] == [{"role": "assistant", "parts": [tool_call], "finish_reason": "tool_call"}]
     assert first["formatterWarnings"] == []
     assert tool["formattedInput"] == [{"role": "assistant", "parts": [tool_call]}]
@@ -206,24 +83,6 @@ def test_thread_returns_gen_ai_spans_in_order(
     assert output_only["formattedInput"] == []
     assert output_only["formattedOutput"] == [{"role": "assistant", "parts": [{"type": "text", "content": "It is sunny in Bangalore."}]}]
     assert output_only["formatterWarnings"] == ["bare assistant text, role assumed"]
-    assert legacy["attributes"]["gen_ai.prompt.0.content"] == "and tomorrow?"
-    assert legacy["formattedInput"] == [{"role": "user", "parts": [{"type": "text", "content": "and tomorrow?"}]}]
-    assert legacy["formattedOutput"] == [{"role": "assistant", "parts": [{"type": "text", "content": "Also sunny."}], "finish_reason": "stop"}]
-    assert flattened["attributes"]["llm.input_messages.0.message.content"] == "thanks"
-    assert flattened["formattedInput"] == [{"role": "user", "parts": [{"type": "text", "content": "thanks"}]}]
-    assert flattened["formattedOutput"] == [{"role": "assistant", "parts": [{"type": "text", "content": "You're welcome."}], "finish_reason": "stop"}]
-    assert responses["formattedInput"] == [{"role": "system", "parts": [{"type": "text", "content": "Be brief."}]}, {"role": "user", "parts": [{"type": "text", "content": "bye"}]}]
-    assert responses["formattedOutput"] == [{"role": "assistant", "parts": [{"type": "text", "content": "Bye!"}], "finish_reason": "stop"}]
-    assert langchain["formattedInput"] == [{"role": "user", "parts": [{"type": "text", "content": "see you"}]}]
-    assert langchain["formattedOutput"] == [{"role": "assistant", "parts": [{"type": "text", "content": "See you!"}], "finish_reason": "stop"}]
-    assert openrouter["formattedInput"] == [{"role": "user", "parts": [{"type": "text", "content": "one word"}]}]
-    assert openrouter["formattedOutput"] == [{"role": "assistant", "parts": [{"type": "text", "content": "Yes."}]}]
-    assert vercel["formattedInput"] == [{"role": "user", "parts": [{"type": "text", "content": "last one"}]}]
-    assert vercel["formattedOutput"] == [{"role": "assistant", "parts": [{"type": "text", "content": "Done."}], "finish_reason": "stop"}]
-    assert anthropic["formattedInput"] == [{"role": "system", "parts": [{"type": "text", "content": "Be terse."}]}, {"role": "user", "parts": [{"type": "text", "content": "ping"}]}]
-    assert anthropic["formattedOutput"] == [{"role": "assistant", "parts": [{"type": "text", "content": "pong"}], "finish_reason": "stop"}]
-    assert gemini["formattedInput"] == [{"role": "user", "parts": [{"type": "text", "content": "ping"}]}]
-    assert gemini["formattedOutput"] == [{"role": "assistant", "parts": [{"type": "text", "content": "pong"}], "finish_reason": "stop"}]
 
 
 def test_thread_paginates_with_cursors(