Compare commits

...

3 Commits

Author SHA1 Message Date
nityanandagohain
255b6fa1ac feat(trace-details): build the span tree and the overview of the trace inspect endpoint
GET /api/v1/traces/{traceID}/inspect now returns the overview. Like the
  waterfall, each call loads the trace's spans with one narrow query and builds
  the tree in memory, with each span's own time, level and the number of spans
  below it. There is no cache and no cap on the number of spans.

  From the tree it returns:
  - identity, with per-service totals and gen_ai usage
  - the spans where each error started, failures not marked as errors, the
    slowest spans by own time, the most expensive gen_ai spans, span links
  - the tree: the first 50 spans, level by level from the root, in time order;
    a trace of up to 50 spans also carries each span's record when they fit

  Every list pages with its nextCursor, and every item carries the span's
  direct parent. A page loads only what its list needs. Span attributes are
  read from the JSON column when it is filled, without the feature flag.

  spanId, search and allSpans answer 501 until the next PRs.

  Refs SigNoz/nerve-pod#651
2026-11-19 21:49:28 +05:30
nityanandagohain
75686f4004 fix(trace-details): update the trace inspect contract for the tree and the span cap 2026-11-19 21:18:30 +05:30
nityanandagohain
2df97d39e2 feat(trace-details): add the API contract for the trace inspect endpoint
Adds GET /api/v1/traces/{traceID}/inspect to the API spec with its query
  parameters and response types. The handler answers 501 for now; the
  implementation follows in separate PRs (overview, open a span, search).

  The response holds one of three parts, named by :
  - overview: identity, errors, failures not marked as errors, slowest spans,
    most expensive spans, span links and the tree
  - span: one span's summary and full record, with its ancestors and descendants
  - find: the spans matching a search, or every span with allSpans

  Every list has total, items and nextCursor, and every item starts with the
  span and its direct parent.

  Refs SigNoz/nerve-pod#651
2026-11-19 16:42:40 +05:30
19 changed files with 4248 additions and 16 deletions

View File

@@ -12680,6 +12680,19 @@ components:
minimum: 0
type: integer
type: object
SpantypesEventV2:
properties:
attributes:
additionalProperties: {}
type: object
isError:
type: boolean
name:
type: string
timeUnixNano:
minimum: 0
type: integer
type: object
SpantypesFieldContext:
enum:
- attribute
@@ -12792,6 +12805,19 @@ components:
required:
- aggregations
type: object
SpantypesGettableTraceInspection:
properties:
find:
$ref: '#/components/schemas/SpantypesTraceSpanList'
overview:
$ref: '#/components/schemas/SpantypesTraceOverview'
span:
$ref: '#/components/schemas/SpantypesTraceSpanDetail'
type:
$ref: '#/components/schemas/SpantypesInspectType'
required:
- type
type: object
SpantypesGettableTraceSummary:
properties:
ai:
@@ -12872,6 +12898,19 @@ components:
nullable: true
type: array
type: object
SpantypesInspectType:
enum:
- overview
- span
- find
type: string
SpantypesLink:
properties:
spanId:
type: string
traceId:
type: string
type: object
SpantypesOtelSpanRef:
properties:
refType:
@@ -13163,6 +13202,100 @@ components:
nullable: true
type: object
type: object
SpantypesSpanRecord:
properties:
attributes:
additionalProperties: {}
type: object
events:
items:
$ref: '#/components/schemas/SpantypesEventV2'
type: array
links:
items:
$ref: '#/components/schemas/SpantypesLink'
type: array
resource:
additionalProperties:
type: string
type: object
spanId:
type: string
statusCode:
type: string
statusMessage:
type: string
truncated:
items:
$ref: '#/components/schemas/SpantypesTruncatedField'
type: array
required:
- spanId
- statusCode
- statusMessage
- attributes
- resource
- events
- links
- truncated
type: object
SpantypesSpanSummary:
properties:
childCount:
minimum: 0
type: integer
durationNano:
minimum: 0
type: integer
hasError:
type: boolean
kind:
type: string
level:
minimum: 0
type: integer
name:
type: string
ownTimeNano:
minimum: 0
type: integer
parentSpanId:
type: string
responseStatusCode:
type: string
serviceName:
type: string
spanId:
type: string
startOffsetNano:
minimum: 0
type: integer
subtreeSpanCount:
minimum: 0
type: integer
required:
- spanId
- name
- serviceName
- kind
- startOffsetNano
- durationNano
- ownTimeNano
- hasError
- level
- childCount
- subtreeSpanCount
type: object
SpantypesSpanTextMatch:
properties:
field:
type: string
snippet:
type: string
required:
- field
- snippet
type: object
SpantypesThreadSpan:
properties:
attributes:
@@ -13229,6 +13362,23 @@ components:
- formattedOutput
- formatterWarnings
type: object
SpantypesTraceAIIdentity:
properties:
modelCallCount:
minimum: 0
type: integer
tokens:
$ref: '#/components/schemas/SpantypesTraceAITokens'
toolCallCount:
minimum: 0
type: integer
totalCost:
type: number
required:
- modelCallCount
- toolCallCount
- tokens
type: object
SpantypesTraceAISummary:
properties:
tokens:
@@ -13262,6 +13412,379 @@ components:
- cacheWrite
- reasoning
type: object
SpantypesTraceErrorItem:
properties:
errorAncestorCount:
minimum: 0
type: integer
parent:
$ref: '#/components/schemas/SpantypesSpanSummary'
span:
$ref: '#/components/schemas/SpantypesSpanSummary'
statusMessage:
type: string
required:
- span
- statusMessage
- errorAncestorCount
type: object
SpantypesTraceErrorList:
properties:
items:
items:
$ref: '#/components/schemas/SpantypesTraceErrorItem'
type: array
nextCursor:
type: string
total:
minimum: 0
type: integer
required:
- total
- items
type: object
SpantypesTraceExpensiveItem:
properties:
cost:
type: number
inputTokens:
minimum: 0
type: integer
outputTokens:
minimum: 0
type: integer
parent:
$ref: '#/components/schemas/SpantypesSpanSummary'
span:
$ref: '#/components/schemas/SpantypesSpanSummary'
required:
- span
- inputTokens
- outputTokens
type: object
SpantypesTraceExpensiveList:
properties:
items:
items:
$ref: '#/components/schemas/SpantypesTraceExpensiveItem'
type: array
nextCursor:
type: string
total:
minimum: 0
type: integer
required:
- total
- items
type: object
SpantypesTraceIdentity:
properties:
ai:
$ref: '#/components/schemas/SpantypesTraceAIIdentity'
durationNano:
minimum: 0
type: integer
errorSpanCount:
minimum: 0
type: integer
hasMissingSpans:
type: boolean
root:
$ref: '#/components/schemas/SpantypesSpanSummary'
rootCount:
minimum: 0
type: integer
serviceCount:
minimum: 0
type: integer
services:
items:
$ref: '#/components/schemas/SpantypesTraceServiceStats'
type: array
spanCount:
minimum: 0
type: integer
startTimestampMillis:
minimum: 0
type: integer
traceId:
type: string
required:
- traceId
- startTimestampMillis
- durationNano
- spanCount
- errorSpanCount
- rootCount
- hasMissingSpans
- serviceCount
- services
type: object
SpantypesTraceLinkItem:
properties:
linkCount:
minimum: 0
type: integer
links:
items:
$ref: '#/components/schemas/SpantypesLink'
type: array
parent:
$ref: '#/components/schemas/SpantypesSpanSummary'
span:
$ref: '#/components/schemas/SpantypesSpanSummary'
required:
- span
- linkCount
- links
type: object
SpantypesTraceLinkList:
properties:
items:
items:
$ref: '#/components/schemas/SpantypesTraceLinkItem'
type: array
nextCursor:
type: string
total:
minimum: 0
type: integer
required:
- total
- items
type: object
SpantypesTraceOverview:
properties:
errors:
$ref: '#/components/schemas/SpantypesTraceErrorList'
expensive:
$ref: '#/components/schemas/SpantypesTraceExpensiveList'
identity:
$ref: '#/components/schemas/SpantypesTraceIdentity'
links:
$ref: '#/components/schemas/SpantypesTraceLinkList'
slowest:
$ref: '#/components/schemas/SpantypesTraceSlowestList'
tree:
$ref: '#/components/schemas/SpantypesTraceTree'
unmarkedFailures:
$ref: '#/components/schemas/SpantypesTraceUnmarkedFailureList'
type: object
SpantypesTraceServiceStats:
properties:
errorSpanCount:
minimum: 0
type: integer
ownTimeNano:
minimum: 0
type: integer
serviceName:
type: string
spanCount:
minimum: 0
type: integer
required:
- serviceName
- spanCount
- errorSpanCount
- ownTimeNano
type: object
SpantypesTraceSlowestItem:
properties:
errorSpanCount:
minimum: 0
type: integer
ownTimeNano:
minimum: 0
type: integer
parent:
$ref: '#/components/schemas/SpantypesSpanSummary'
span:
$ref: '#/components/schemas/SpantypesSpanSummary'
spanCount:
minimum: 0
type: integer
required:
- span
- spanCount
- ownTimeNano
- errorSpanCount
type: object
SpantypesTraceSlowestList:
properties:
items:
items:
$ref: '#/components/schemas/SpantypesTraceSlowestItem'
type: array
nextCursor:
type: string
total:
minimum: 0
type: integer
required:
- total
- items
type: object
SpantypesTraceSpanDetail:
properties:
ancestorCount:
minimum: 0
type: integer
ancestors:
items:
$ref: '#/components/schemas/SpantypesSpanSummary'
type: array
descendants:
items:
$ref: '#/components/schemas/SpantypesTraceSpanLevel'
type: array
percentile:
$ref: '#/components/schemas/SpantypesTraceSpanPercentile'
record:
$ref: '#/components/schemas/SpantypesSpanRecord'
summary:
$ref: '#/components/schemas/SpantypesSpanSummary'
required:
- summary
- record
- ancestors
- ancestorCount
- descendants
type: object
SpantypesTraceSpanLevel:
properties:
level:
minimum: 0
type: integer
spans:
items:
$ref: '#/components/schemas/SpantypesSpanSummary'
type: array
total:
minimum: 0
type: integer
required:
- level
- total
- spans
type: object
SpantypesTraceSpanList:
properties:
items:
items:
$ref: '#/components/schemas/SpantypesTraceSpanMatch'
type: array
nextCursor:
type: string
total:
minimum: 0
type: integer
required:
- total
- items
type: object
SpantypesTraceSpanMatch:
properties:
matches:
items:
$ref: '#/components/schemas/SpantypesSpanTextMatch'
type: array
parent:
$ref: '#/components/schemas/SpantypesSpanSummary'
span:
$ref: '#/components/schemas/SpantypesSpanSummary'
required:
- span
- matches
type: object
SpantypesTraceSpanPercentile:
properties:
p50Nano:
minimum: 0
type: integer
p90Nano:
minimum: 0
type: integer
p99Nano:
minimum: 0
type: integer
position:
format: double
type: number
windowMinutes:
minimum: 0
type: integer
required:
- p50Nano
- p90Nano
- p99Nano
- position
- windowMinutes
type: object
SpantypesTraceTree:
properties:
complete:
type: boolean
records:
items:
$ref: '#/components/schemas/SpantypesSpanRecord'
type: array
spans:
items:
$ref: '#/components/schemas/SpantypesSpanSummary'
type: array
total:
minimum: 0
type: integer
required:
- total
- complete
- spans
type: object
SpantypesTraceUnmarkedFailureItem:
properties:
parent:
$ref: '#/components/schemas/SpantypesSpanSummary'
reason:
$ref: '#/components/schemas/SpantypesUnmarkedFailureReason'
span:
$ref: '#/components/schemas/SpantypesSpanSummary'
required:
- span
- reason
type: object
SpantypesTraceUnmarkedFailureList:
properties:
items:
items:
$ref: '#/components/schemas/SpantypesTraceUnmarkedFailureItem'
type: array
nextCursor:
type: string
total:
minimum: 0
type: integer
required:
- total
- items
type: object
SpantypesTruncatedField:
properties:
field:
type: string
length:
minimum: 0
type: integer
required:
- field
- length
type: object
SpantypesUnmarkedFailureReason:
enum:
- http_4xx
- grpc_status
- exception_event
type: string
SpantypesUpdatableSpanMapper:
properties:
config:
@@ -19284,6 +19807,150 @@ paths:
tags:
- tracedetail
x-signoz-stability: alpha
/api/v1/traces/{traceID}/inspect:
get:
deprecated: false
description: 'Returns one of three parts of a trace of any size, named by type,
each within 50,000 characters. Without spanId or search it returns the overview:
the trace''s identity with per-service totals, the spans where each error
started, the failures not marked as errors (a 4xx response, a gRPC status
other than OK, an exception event), the slowest spans by own time with same-name
spans under one parent taken together, the most expensive gen_ai spans, and
the span tree. The tree holds the first 50 spans of the trace: level by level
from the root, and in time order on the level that does not fit. A trace of
up to 50 spans also returns each span''s full record when they fit in 20,000
characters. With spanId it returns that span''s summary and full record, its
ancestors, and the spans below it level by level, at most 20 per level; values
over 500 characters are cut and listed in truncated. With search it returns
the matching spans, each with where the words were found; allSpans returns
every span instead. Every list carries its total, and every list item the
span''s direct parent. Pass a nextCursor as cursor with the same parameters
to get the next page.'
operationId: InspectTrace
parameters:
- description: Returns this span in full with the spans around it. Cannot be
combined with search.
in: query
name: spanId
schema:
description: Returns this span in full with the spans around it. Cannot
be combined with search.
type: string
- description: 'Returns the spans that match, earliest first: words, a filter
expression, or both, such as worldpay AND has_error = true. Words match
the span name, status message, attribute values and events, ignoring case.'
in: query
name: search
schema:
description: 'Returns the spans that match, earliest first: words, a filter
expression, or both, such as worldpay AND has_error = true. Words match
the span name, status message, attribute values and events, ignoring case.'
type: string
- description: With spanId, the number of levels of ancestors and of descendants
returned, 1 to 10; 2 when not given. The root span is always returned.
in: query
name: depth
schema:
description: With spanId, the number of levels of ancestors and of descendants
returned, 1 to 10; 2 when not given. The root span is always returned.
type: integer
- description: 'Returns a different response: every span of the trace as a short
line, in tree order, or with spanId that span and every span below it. Cannot
be combined with search or depth.'
in: query
name: allSpans
schema:
description: 'Returns a different response: every span of the trace as a
short line, in tree order, or with spanId that span and every span below
it. Cannot be combined with search or depth.'
type: boolean
- description: With spanId, attribute names returned without the 500 character
cut; the record then holds only these attributes.
in: query
name: selectFields
schema:
description: With spanId, attribute names returned without the 500 character
cut; the record then holds only these attributes.
items:
type: string
type: array
- description: Page size for search and for allSpans, at most 1000; when not
given, 20 with search and 400 with allSpans. A page holds fewer spans when
it would pass the 50,000 character limit.
in: query
name: limit
schema:
description: Page size for search and for allSpans, at most 1000; when not
given, 20 with search and 400 with allSpans. A page holds fewer spans
when it would pass the 50,000 character limit.
type: integer
- description: The nextCursor of an earlier response to the same request; returns
the next page.
in: query
name: cursor
schema:
description: The nextCursor of an earlier response to the same request;
returns the next page.
type: string
- in: path
name: traceID
required: true
schema:
type: string
responses:
"200":
content:
application/json:
schema:
properties:
data:
$ref: '#/components/schemas/SpantypesGettableTraceInspection'
status:
type: string
required:
- status
- data
type: object
description: OK
"400":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Bad Request
"401":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Unauthorized
"403":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
"404":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Not Found
"500":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Internal Server Error
security:
- api_key:
- VIEWER
- tokenizer:
- VIEWER
summary: Inspect a trace
tags:
- tracedetail
x-signoz-stability: alpha
/api/v1/traces/{traceID}/summary:
get:
deprecated: false

View File

@@ -13817,6 +13817,28 @@ export interface SpantypesEventDTO {
timeUnixNano?: number;
}
export type SpantypesEventV2DTOAttributes = { [key: string]: unknown };
export interface SpantypesEventV2DTO {
/**
* @type object
*/
attributes?: SpantypesEventV2DTOAttributes;
/**
* @type boolean
*/
isError?: boolean;
/**
* @type string
*/
name?: string;
/**
* @type integer
* @minimum 0
*/
timeUnixNano?: number;
}
export enum SpantypesFieldContextDTO {
attribute = 'attribute',
resource = 'resource',
@@ -14136,6 +14158,168 @@ export interface SpantypesGettableTraceAggregationsDTO {
aggregations: SpantypesSpanAggregationResultDTO[];
}
export interface SpantypesSpanTextMatchDTO {
/**
* @type string
*/
field: string;
/**
* @type string
*/
snippet: string;
}
export interface SpantypesSpanSummaryDTO {
/**
* @type integer
* @minimum 0
*/
childCount: number;
/**
* @type integer
* @minimum 0
*/
durationNano: number;
/**
* @type boolean
*/
hasError: boolean;
/**
* @type string
*/
kind: string;
/**
* @type integer
* @minimum 0
*/
level: number;
/**
* @type string
*/
name: string;
/**
* @type integer
* @minimum 0
*/
ownTimeNano: number;
/**
* @type string
*/
parentSpanId?: string;
/**
* @type string
*/
responseStatusCode?: string;
/**
* @type string
*/
serviceName: string;
/**
* @type string
*/
spanId: string;
/**
* @type integer
* @minimum 0
*/
startOffsetNano: number;
/**
* @type integer
* @minimum 0
*/
subtreeSpanCount: number;
}
export interface SpantypesTraceSpanMatchDTO {
/**
* @type array
*/
matches: SpantypesSpanTextMatchDTO[];
parent?: SpantypesSpanSummaryDTO;
span: SpantypesSpanSummaryDTO;
}
export interface SpantypesTraceSpanListDTO {
/**
* @type array
*/
items: SpantypesTraceSpanMatchDTO[];
/**
* @type string
*/
nextCursor?: string;
/**
* @type integer
* @minimum 0
*/
total: number;
}
export interface SpantypesTraceErrorItemDTO {
/**
* @type integer
* @minimum 0
*/
errorAncestorCount: number;
parent?: SpantypesSpanSummaryDTO;
span: SpantypesSpanSummaryDTO;
/**
* @type string
*/
statusMessage: string;
}
export interface SpantypesTraceErrorListDTO {
/**
* @type array
*/
items: SpantypesTraceErrorItemDTO[];
/**
* @type string
*/
nextCursor?: string;
/**
* @type integer
* @minimum 0
*/
total: number;
}
export interface SpantypesTraceExpensiveItemDTO {
/**
* @type number
*/
cost?: number;
/**
* @type integer
* @minimum 0
*/
inputTokens: number;
/**
* @type integer
* @minimum 0
*/
outputTokens: number;
parent?: SpantypesSpanSummaryDTO;
span: SpantypesSpanSummaryDTO;
}
export interface SpantypesTraceExpensiveListDTO {
/**
* @type array
*/
items: SpantypesTraceExpensiveItemDTO[];
/**
* @type string
*/
nextCursor?: string;
/**
* @type integer
* @minimum 0
*/
total: number;
}
export interface SpantypesTraceAITokensDTO {
/**
* @type integer
@@ -14164,6 +14348,354 @@ export interface SpantypesTraceAITokensDTO {
reasoning: number;
}
export interface SpantypesTraceAIIdentityDTO {
/**
* @type integer
* @minimum 0
*/
modelCallCount: number;
tokens: SpantypesTraceAITokensDTO;
/**
* @type integer
* @minimum 0
*/
toolCallCount: number;
/**
* @type number
*/
totalCost?: number;
}
export interface SpantypesTraceServiceStatsDTO {
/**
* @type integer
* @minimum 0
*/
errorSpanCount: number;
/**
* @type integer
* @minimum 0
*/
ownTimeNano: number;
/**
* @type string
*/
serviceName: string;
/**
* @type integer
* @minimum 0
*/
spanCount: number;
}
export interface SpantypesTraceIdentityDTO {
ai?: SpantypesTraceAIIdentityDTO;
/**
* @type integer
* @minimum 0
*/
durationNano: number;
/**
* @type integer
* @minimum 0
*/
errorSpanCount: number;
/**
* @type boolean
*/
hasMissingSpans: boolean;
root?: SpantypesSpanSummaryDTO;
/**
* @type integer
* @minimum 0
*/
rootCount: number;
/**
* @type integer
* @minimum 0
*/
serviceCount: number;
/**
* @type array
*/
services: SpantypesTraceServiceStatsDTO[];
/**
* @type integer
* @minimum 0
*/
spanCount: number;
/**
* @type integer
* @minimum 0
*/
startTimestampMillis: number;
/**
* @type string
*/
traceId: string;
}
export interface SpantypesLinkDTO {
/**
* @type string
*/
spanId?: string;
/**
* @type string
*/
traceId?: string;
}
export interface SpantypesTraceLinkItemDTO {
/**
* @type integer
* @minimum 0
*/
linkCount: number;
/**
* @type array
*/
links: SpantypesLinkDTO[];
parent?: SpantypesSpanSummaryDTO;
span: SpantypesSpanSummaryDTO;
}
export interface SpantypesTraceLinkListDTO {
/**
* @type array
*/
items: SpantypesTraceLinkItemDTO[];
/**
* @type string
*/
nextCursor?: string;
/**
* @type integer
* @minimum 0
*/
total: number;
}
export interface SpantypesTraceSlowestItemDTO {
/**
* @type integer
* @minimum 0
*/
errorSpanCount: number;
/**
* @type integer
* @minimum 0
*/
ownTimeNano: number;
parent?: SpantypesSpanSummaryDTO;
span: SpantypesSpanSummaryDTO;
/**
* @type integer
* @minimum 0
*/
spanCount: number;
}
export interface SpantypesTraceSlowestListDTO {
/**
* @type array
*/
items: SpantypesTraceSlowestItemDTO[];
/**
* @type string
*/
nextCursor?: string;
/**
* @type integer
* @minimum 0
*/
total: number;
}
export interface SpantypesTruncatedFieldDTO {
/**
* @type string
*/
field: string;
/**
* @type integer
* @minimum 0
*/
length: number;
}
export type SpantypesSpanRecordDTOAttributes = { [key: string]: unknown };
export type SpantypesSpanRecordDTOResource = { [key: string]: string };
export interface SpantypesSpanRecordDTO {
/**
* @type object
*/
attributes: SpantypesSpanRecordDTOAttributes;
/**
* @type array
*/
events: SpantypesEventV2DTO[];
/**
* @type array
*/
links: SpantypesLinkDTO[];
/**
* @type object
*/
resource: SpantypesSpanRecordDTOResource;
/**
* @type string
*/
spanId: string;
/**
* @type string
*/
statusCode: string;
/**
* @type string
*/
statusMessage: string;
/**
* @type array
*/
truncated: SpantypesTruncatedFieldDTO[];
}
export interface SpantypesTraceTreeDTO {
/**
* @type boolean
*/
complete: boolean;
/**
* @type array
*/
records?: SpantypesSpanRecordDTO[];
/**
* @type array
*/
spans: SpantypesSpanSummaryDTO[];
/**
* @type integer
* @minimum 0
*/
total: number;
}
export enum SpantypesUnmarkedFailureReasonDTO {
http_4xx = 'http_4xx',
grpc_status = 'grpc_status',
exception_event = 'exception_event',
}
export interface SpantypesTraceUnmarkedFailureItemDTO {
parent?: SpantypesSpanSummaryDTO;
reason: SpantypesUnmarkedFailureReasonDTO;
span: SpantypesSpanSummaryDTO;
}
export interface SpantypesTraceUnmarkedFailureListDTO {
/**
* @type array
*/
items: SpantypesTraceUnmarkedFailureItemDTO[];
/**
* @type string
*/
nextCursor?: string;
/**
* @type integer
* @minimum 0
*/
total: number;
}
export interface SpantypesTraceOverviewDTO {
errors?: SpantypesTraceErrorListDTO;
expensive?: SpantypesTraceExpensiveListDTO;
identity?: SpantypesTraceIdentityDTO;
links?: SpantypesTraceLinkListDTO;
slowest?: SpantypesTraceSlowestListDTO;
tree?: SpantypesTraceTreeDTO;
unmarkedFailures?: SpantypesTraceUnmarkedFailureListDTO;
}
export interface SpantypesTraceSpanLevelDTO {
/**
* @type integer
* @minimum 0
*/
level: number;
/**
* @type array
*/
spans: SpantypesSpanSummaryDTO[];
/**
* @type integer
* @minimum 0
*/
total: number;
}
export interface SpantypesTraceSpanPercentileDTO {
/**
* @type integer
* @minimum 0
*/
p50Nano: number;
/**
* @type integer
* @minimum 0
*/
p90Nano: number;
/**
* @type integer
* @minimum 0
*/
p99Nano: number;
/**
* @type number
* @format double
*/
position: number;
/**
* @type integer
* @minimum 0
*/
windowMinutes: number;
}
export interface SpantypesTraceSpanDetailDTO {
/**
* @type integer
* @minimum 0
*/
ancestorCount: number;
/**
* @type array
*/
ancestors: SpantypesSpanSummaryDTO[];
/**
* @type array
*/
descendants: SpantypesTraceSpanLevelDTO[];
percentile?: SpantypesTraceSpanPercentileDTO;
record: SpantypesSpanRecordDTO;
summary: SpantypesSpanSummaryDTO;
}
export enum SpantypesInspectTypeDTO {
overview = 'overview',
span = 'span',
find = 'find',
}
export interface SpantypesGettableTraceInspectionDTO {
find?: SpantypesTraceSpanListDTO;
overview?: SpantypesTraceOverviewDTO;
span?: SpantypesTraceSpanDetailDTO;
type: SpantypesInspectTypeDTO;
}
export interface SpantypesTraceAISummaryDTO {
tokens: SpantypesTraceAITokensDTO;
/**
@@ -16035,6 +16567,55 @@ export type GetTraceAggregations200 = {
status: string;
};
export type InspectTracePathParameters = {
traceID: string;
};
export type InspectTraceParams = {
/**
* @type string
* @description Returns this span in full with the spans around it. Cannot be combined with search.
*/
spanId?: string;
/**
* @type string
* @description Returns the spans that match, earliest first: words, a filter expression, or both, such as worldpay AND has_error = true. Words match the span name, status message, attribute values and events, ignoring case.
*/
search?: string;
/**
* @type integer
* @description With spanId, the number of levels of ancestors and of descendants returned, 1 to 10; 2 when not given. The root span is always returned.
*/
depth?: number;
/**
* @type boolean
* @description Returns a different response: every span of the trace as a short line, in tree order, or with spanId that span and every span below it. Cannot be combined with search or depth.
*/
allSpans?: boolean;
/**
* @type array
* @description With spanId, attribute names returned without the 500 character cut; the record then holds only these attributes.
*/
selectFields?: string[];
/**
* @type integer
* @description Page size for search and for allSpans, at most 1000; when not given, 20 with search and 400 with allSpans. A page holds fewer spans when it would pass the 50,000 character limit.
*/
limit?: number;
/**
* @type string
* @description The nextCursor of an earlier response to the same request; returns the next page.
*/
cursor?: string;
};
export type InspectTrace200 = {
data: SpantypesGettableTraceInspectionDTO;
/**
* @type string
*/
status: string;
};
export type GetTraceSummaryPathParameters = {
traceID: string;
};

View File

@@ -29,6 +29,9 @@ import type {
GetTraceThreadPathParameters,
GetWaterfallV4200,
GetWaterfallV4PathParameters,
InspectTrace200,
InspectTraceParams,
InspectTracePathParameters,
RenderErrorResponseDTO,
SpantypesPostableFlamegraphDTO,
SpantypesPostableTraceAggregationsDTO,
@@ -158,6 +161,117 @@ export const useGetTraceAggregations = <
> => {
return useMutation(getGetTraceAggregationsMutationOptions(options));
};
/**
* Returns one of three parts of a trace of any size, named by type, each within 50,000 characters. Without spanId or search it returns the overview: the trace's identity with per-service totals, the spans where each error started, the failures not marked as errors (a 4xx response, a gRPC status other than OK, an exception event), the slowest spans by own time with same-name spans under one parent taken together, the most expensive gen_ai spans, and the span tree. The tree holds the first 50 spans of the trace: level by level from the root, and in time order on the level that does not fit. A trace of up to 50 spans also returns each span's full record when they fit in 20,000 characters. With spanId it returns that span's summary and full record, its ancestors, and the spans below it level by level, at most 20 per level; values over 500 characters are cut and listed in truncated. With search it returns the matching spans, each with where the words were found; allSpans returns every span instead. Every list carries its total, and every list item the span's direct parent. Pass a nextCursor as cursor with the same parameters to get the next page.
* @summary Inspect a trace
*/
export const inspectTrace = (
{ traceID }: InspectTracePathParameters,
params?: InspectTraceParams,
signal?: AbortSignal,
) => {
return GeneratedAPIInstance<InspectTrace200>({
url: `/api/v1/traces/${traceID}/inspect`,
method: 'GET',
params,
signal,
});
};
export const getInspectTraceQueryKey = (
{ traceID }: InspectTracePathParameters,
params?: InspectTraceParams,
) => {
return [
`/api/v1/traces/${traceID}/inspect`,
...(params ? [params] : []),
] as const;
};
export const getInspectTraceQueryOptions = <
TData = Awaited<ReturnType<typeof inspectTrace>>,
TError = ErrorType<RenderErrorResponseDTO>,
>(
{ traceID }: InspectTracePathParameters,
params?: InspectTraceParams,
options?: {
query?: UseQueryOptions<
Awaited<ReturnType<typeof inspectTrace>>,
TError,
TData
>;
},
) => {
const { query: queryOptions } = options ?? {};
const queryKey =
queryOptions?.queryKey ?? getInspectTraceQueryKey({ traceID }, params);
const queryFn: QueryFunction<Awaited<ReturnType<typeof inspectTrace>>> = ({
signal,
}) => inspectTrace({ traceID }, params, signal);
return {
queryKey,
queryFn,
enabled: traceID !== null && traceID !== undefined,
...queryOptions,
} as UseQueryOptions<
Awaited<ReturnType<typeof inspectTrace>>,
TError,
TData
> & { queryKey: QueryKey };
};
export type InspectTraceQueryResult = NonNullable<
Awaited<ReturnType<typeof inspectTrace>>
>;
export type InspectTraceQueryError = ErrorType<RenderErrorResponseDTO>;
/**
* @summary Inspect a trace
*/
export function useInspectTrace<
TData = Awaited<ReturnType<typeof inspectTrace>>,
TError = ErrorType<RenderErrorResponseDTO>,
>(
{ traceID }: InspectTracePathParameters,
params?: InspectTraceParams,
options?: {
query?: UseQueryOptions<
Awaited<ReturnType<typeof inspectTrace>>,
TError,
TData
>;
},
): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
const queryOptions = getInspectTraceQueryOptions({ traceID }, params, options);
const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
queryKey: QueryKey;
};
return withQueryKey(query, queryOptions.queryKey);
}
/**
* @summary Inspect a trace
*/
export const invalidateInspectTrace = async (
queryClient: QueryClient,
{ traceID }: InspectTracePathParameters,
params?: InspectTraceParams,
options?: InvalidateOptions,
): Promise<QueryClient> => {
await queryClient.invalidateQueries(
{ queryKey: getInspectTraceQueryKey({ traceID }, params) },
options,
);
return queryClient;
};
/**
* Returns the trace's time range, root span, span and error counts, and whether any spans are missing. AI traces also include token and cost totals.
* @summary Get summary for a trace

View File

@@ -102,5 +102,23 @@ func (provider *provider) addTraceDetailRoutes(router *mux.Router) error {
return err
}
if err := router.Handle("/api/v1/traces/{traceID}/inspect", handler.New(
provider.authzMiddleware.ViewAccess(provider.traceDetailHandler.InspectTrace),
handler.OpenAPIDef{
ID: "InspectTrace",
Tags: []string{"tracedetail"},
Summary: "Inspect a trace",
Description: "Returns one of three parts of a trace of any size, named by type, each within 50,000 characters. Without spanId or search it returns the overview: the trace's identity with per-service totals, the spans where each error started, the failures not marked as errors (a 4xx response, a gRPC status other than OK, an exception event), the slowest spans by own time with same-name spans under one parent taken together, the most expensive gen_ai spans, and the span tree. The tree holds the first 50 spans of the trace: level by level from the root, and in time order on the level that does not fit. A trace of up to 50 spans also returns each span's full record when they fit in 20,000 characters. With spanId it returns that span's summary and full record, its ancestors, and the spans below it level by level, at most 20 per level; values over 500 characters are cut and listed in truncated. With search it returns the matching spans, each with where the words were found; allSpans returns every span instead. Every list carries its total, and every list item the span's direct parent. Pass a nextCursor as cursor with the same parameters to get the next page.",
RequestQuery: new(spantypes.InspectTraceParams),
Response: new(spantypes.GettableTraceInspection),
ResponseContentType: "application/json",
SuccessStatusCode: http.StatusOK,
ErrorStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound},
SecuritySchemes: newSecuritySchemes(types.RoleViewer),
},
)).Methods(http.MethodGet).GetError(); err != nil {
return err
}
return nil
}

View File

@@ -127,3 +127,31 @@ func (h *handler) GetThread(rw http.ResponseWriter, r *http.Request) {
render.Success(rw, http.StatusOK, result)
}
func (h *handler) InspectTrace(rw http.ResponseWriter, r *http.Request) {
claims, err := authtypes.ClaimsFromContext(r.Context())
if err != nil {
render.Error(rw, err)
return
}
params := new(spantypes.InspectTraceParams)
if err := binding.Query.BindQuery(r.URL.Query(), params); err != nil {
render.Error(rw, err)
return
}
query, err := spantypes.NewInspectQuery(params)
if err != nil {
render.Error(rw, err)
return
}
result, err := h.module.InspectTrace(r.Context(), valuer.MustNewUUID(claims.OrgID), mux.Vars(r)["traceID"], query)
if err != nil {
render.Error(rw, err)
return
}
render.Success(rw, http.StatusOK, result)
}

View File

@@ -11,6 +11,7 @@ import (
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
"go.opentelemetry.io/otel/metric"
"golang.org/x/sync/errgroup"
)
// formattedSpansLogThreshold is the number of formatted gen_ai spans in one waterfall response above which
@@ -292,3 +293,113 @@ func (m *module) getWindowedFlamegraph(ctx context.Context, traceID, selectedSpa
enrichedSpans := flamegraphTrace.EnrichSelectedSpans(selectedSpans, fullSpans, selectFields)
return spantypes.NewGettableFlamegraphTrace(enrichedSpans, start, end, true), nil
}
func (m *module) InspectTrace(ctx context.Context, orgID valuer.UUID, traceID string, query *spantypes.InspectQuery) (*spantypes.GettableTraceInspection, error) {
overview, err := m.getTraceOverview(ctx, orgID, traceID, query)
if err != nil {
return nil, err
}
return &spantypes.GettableTraceInspection{Type: query.Type, Overview: overview}, nil
}
func (m *module) getTraceOverview(ctx context.Context, orgID valuer.UUID, traceID string, query *spantypes.InspectQuery) (*spantypes.TraceOverview, error) {
bounds, err := m.store.GetTraceBounds(ctx, traceID)
if err != nil {
return nil, err
}
// A page of one list loads only what that list is built from.
withAISpans := query.List == "" || query.List == spantypes.OverviewListExpensive
withLinkedSpans := query.List == "" || query.List == spantypes.OverviewListLinks
skeleton, err := m.loadSkeleton(ctx, orgID, traceID, bounds, withAISpans, withLinkedSpans)
if err != nil {
return nil, err
}
switch query.List {
case spantypes.OverviewListErrors:
errorList, err := m.errorPage(ctx, traceID, bounds, skeleton, query.Offset)
if err != nil {
return nil, err
}
return &spantypes.TraceOverview{Errors: errorList}, nil
case spantypes.OverviewListUnmarkedFailures:
return &spantypes.TraceOverview{UnmarkedFailures: skeleton.UnmarkedFailurePage(query.Offset)}, nil
case spantypes.OverviewListSlowest:
return &spantypes.TraceOverview{Slowest: skeleton.SlowestPage(query.Offset)}, nil
case spantypes.OverviewListExpensive:
return &spantypes.TraceOverview{Expensive: skeleton.ExpensivePage(query.Offset)}, nil
case spantypes.OverviewListLinks:
return &spantypes.TraceOverview{Links: skeleton.LinkPage(query.Offset)}, nil
}
errorList, err := m.errorPage(ctx, traceID, bounds, skeleton, 0)
if err != nil {
return nil, err
}
overview := &spantypes.TraceOverview{
Identity: skeleton.Identity(),
Errors: errorList,
UnmarkedFailures: skeleton.UnmarkedFailurePage(0),
Slowest: skeleton.SlowestPage(0),
Expensive: skeleton.ExpensivePage(0),
Links: skeleton.LinkPage(0),
Tree: skeleton.Tree(),
}
if spantypes.IsSmallTrace(skeleton.Len()) {
spans, err := m.store.GetTraceSpans(ctx, traceID, bounds)
if err != nil {
return nil, err
}
if records, ok := spantypes.SmallTraceRecords(spans); ok {
overview.Tree.Records = records
}
}
skeleton.FitOverview(overview)
return overview, nil
}
func (m *module) errorPage(ctx context.Context, traceID string, bounds *spantypes.TraceBounds, skeleton *spantypes.TraceSkeleton, offset int) (*spantypes.TraceErrorList, error) {
errorList, spanIDs := skeleton.ErrorPage(offset)
messages, err := m.store.GetSpanStatusMessages(ctx, traceID, bounds, spanIDs)
if err != nil {
return nil, err
}
errorList.SetStatusMessages(messages)
return errorList, nil
}
// loadSkeleton builds the span tree of the trace, with the gen_ai usage and the span links when asked for.
func (m *module) loadSkeleton(ctx context.Context, orgID valuer.UUID, traceID string, bounds *spantypes.TraceBounds, withAISpans, withLinkedSpans bool) (*spantypes.TraceSkeleton, error) {
var (
skeleton *spantypes.TraceSkeleton
aiSpans []spantypes.AISpanUsage
linkedSpans []spantypes.LinkedSpan
)
group, groupCtx := errgroup.WithContext(ctx)
group.Go(func() error {
var err error
skeleton, err = m.store.GetTraceSkeleton(groupCtx, traceID, bounds)
return err
})
if withAISpans {
group.Go(func() error {
var err error
aiSpans, err = m.store.GetAISpanUsage(groupCtx, orgID, traceID, bounds)
return err
})
}
if withLinkedSpans {
group.Go(func() error {
var err error
linkedSpans, err = m.store.GetLinkedSpans(groupCtx, traceID, bounds)
return err
})
}
if err := group.Wait(); err != nil {
return nil, err
}
if skeleton.Len() == 0 {
return nil, spantypes.ErrTraceNotFound
}
return skeleton.WithAISpans(aiSpans).WithLinkedSpans(linkedSpans), nil
}

View File

@@ -24,6 +24,15 @@ import (
const colServiceName = `resource_string_service$$$$name` // $ gets escaped so $$$$ converts to $$.
var genAIUsageValues = []struct{ key, alias string }{
{aiobservabilitytypes.GenAIUsageInputTokens, "input_tokens_value"},
{aiobservabilitytypes.GenAIUsageOutputTokens, "output_tokens_value"},
{aiobservabilitytypes.GenAIUsageCacheReadInputTokens, "cache_read_tokens_value"},
{aiobservabilitytypes.GenAIUsageCacheCreationInputTokens, "cache_write_tokens_value"},
{aiobservabilitytypes.GenAIUsageReasoningOutputTokens, "reasoning_tokens_value"},
{aiobservabilitytypes.SignozGenAITotalCost, "total_cost_value"},
}
// storableSpanColumns are the columns a waterfall span scans. A row's legacy maps are blanked when its
// JSON holds the attributes.
var storableSpanColumns = []string{
@@ -172,20 +181,7 @@ func (s *traceStore) GetTraceStats(ctx context.Context, orgID valuer.UUID, trace
// genAISpanColumns returns the gen_ai columns aggregated per span, resolved across attribute evolutions.
func (s *traceStore) genAISpanColumns(ctx context.Context, orgID valuer.UUID, bounds *spantypes.TraceBounds, sb *sqlbuilder.SelectBuilder) ([]string, error) {
values := []struct{ key, alias string }{
{aiobservabilitytypes.GenAIUsageInputTokens, "input_tokens_value"},
{aiobservabilitytypes.GenAIUsageOutputTokens, "output_tokens_value"},
{aiobservabilitytypes.GenAIUsageCacheReadInputTokens, "cache_read_tokens_value"},
{aiobservabilitytypes.GenAIUsageCacheCreationInputTokens, "cache_write_tokens_value"},
{aiobservabilitytypes.GenAIUsageReasoningOutputTokens, "reasoning_tokens_value"},
{aiobservabilitytypes.SignozGenAITotalCost, "total_cost_value"},
}
names := slices.Clone(aiobservabilitytypes.GenAISpanGateKeys)
for _, value := range values {
names = append(names, value.key)
}
keys, err := s.genAIFieldKeys(ctx, orgID, names)
keys, err := s.genAIFieldKeys(ctx, orgID, genAIUsageKeyNames())
if err != nil {
return nil, err
}
@@ -195,9 +191,26 @@ func (s *traceStore) genAISpanColumns(ctx context.Context, orgID valuer.UUID, bo
if err != nil {
return nil, err
}
columns := []string{isGenAI + " AS is_gen_ai"}
usage, err := s.genAIUsageColumns(ctx, q, keys)
if err != nil {
return nil, err
}
return append([]string{isGenAI + " AS is_gen_ai"}, usage...), nil
}
for _, value := range values {
// genAIUsageKeyNames lists the gate keys and the usage keys a gen_ai query resolves.
func genAIUsageKeyNames() []string {
names := slices.Clone(aiobservabilitytypes.GenAISpanGateKeys)
for _, value := range genAIUsageValues {
names = append(names, value.key)
}
return names
}
// genAIUsageColumns returns the token and cost columns of a span, resolved across attribute evolutions.
func (s *traceStore) genAIUsageColumns(ctx context.Context, q qbtypes.QueryInfo, keys map[string][]*telemetrytypes.TelemetryFieldKey) ([]string, error) {
columns := make([]string, 0, len(genAIUsageValues))
for _, value := range genAIUsageValues {
// lookup by number, the type metadata stores numeric attributes under; float64 is only the output cast
// fixed gen_ai keys that many traces lack, so a not-found warning means nothing here
expr, _, err := querybuilder.ResolveColumn(ctx, q, s.storage, attributeKey(value.key, telemetrytypes.FieldDataTypeNumber), telemetrytypes.FieldDataTypeFloat64, keys)
@@ -523,3 +536,160 @@ func (s *traceStore) anyExistsCondition(ctx context.Context, q qbtypes.QueryInfo
}
return sb.Or(conds...), nil
}
// The collector marks an exception event with isError when it writes the span.
const colHasExceptionEvent = `(NOT has_error AND arrayExists(e -> position(e, '"isError":true') > 0, events)) AS has_exception_event`
type spanStatusMessageRow struct {
SpanID string `ch:"span_id"`
StatusMessage string `ch:"status_message"`
}
type spanLinksRow struct {
SpanID string `ch:"span_id"`
Links string `ch:"links"`
}
func (s *traceStore) GetTraceSkeleton(ctx context.Context, traceID string, bounds *spantypes.TraceBounds) (*spantypes.TraceSkeleton, error) {
sb := sqlbuilder.NewSelectBuilder()
sb.Select(
"DISTINCT ON (span_id) span_id", "parent_span_id", "timestamp", "duration_nano", "has_error",
"name", colServiceName, "kind_string", "response_status_code", colHasExceptionEvent,
)
sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
sb.Where(
sb.E("trace_id", traceID),
sb.GE("ts_bucket_start", bounds.Start.Unix()-1800),
sb.LE("ts_bucket_start", bounds.End.Unix()),
)
sb.OrderByAsc("timestamp")
sb.OrderByAsc("name")
sb.OrderByAsc("span_id")
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
rows, err := s.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
if err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error querying trace skeleton")
}
defer rows.Close()
builder := spantypes.NewTraceSkeletonBuilder(traceID, int(bounds.NumSpans))
var span spantypes.SkeletonSpan
// Scanning into one reused list of fields, not ScanStruct, keeps the per-row cost to the strings.
fields := []any{
&span.SpanID, &span.ParentSpanID, &span.StartTime, &span.DurationNano, &span.HasError,
&span.Name, &span.ServiceName, &span.KindString, &span.ResponseStatusCode, &span.HasExceptionEvent,
}
for rows.Next() {
if err := rows.Scan(fields...); err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error scanning trace skeleton")
}
builder.Add(&span)
}
if err := rows.Err(); err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error reading trace skeleton")
}
return builder.Build(), nil
}
func (s *traceStore) GetAISpanUsage(ctx context.Context, orgID valuer.UUID, traceID string, bounds *spantypes.TraceBounds) ([]spantypes.AISpanUsage, error) {
keys, err := s.genAIFieldKeys(ctx, orgID, genAIUsageKeyNames())
if err != nil {
return nil, err
}
sb := sqlbuilder.NewSelectBuilder()
q := querybuilder.NewQueryInfo(ctx, orgID, s.flagger, telemetrytypes.SignalTraces, nil, uint64(bounds.Start.UnixNano()), uint64(bounds.End.UnixNano()))
// Like the waterfall, read the JSON attributes wherever that column is registered for the trace's
// time range and the legacy maps otherwise, without waiting for the use_trace_attributes_json flag.
q.TraceAttrsJSONOn = true
isGenAI, err := s.anyExistsCondition(ctx, q, aiobservabilitytypes.GenAISpanGateKeys, keys, sb)
if err != nil {
return nil, err
}
isModelCall, err := s.anyExistsCondition(ctx, q, []string{aiobservabilitytypes.GenAIRequestModel}, keys, sb)
if err != nil {
return nil, err
}
isToolCall, err := s.anyExistsCondition(ctx, q, []string{aiobservabilitytypes.GenAIToolName}, keys, sb)
if err != nil {
return nil, err
}
usage, err := s.genAIUsageColumns(ctx, q, keys)
if err != nil {
return nil, err
}
sb.Select("DISTINCT ON (span_id) span_id", isModelCall+" AS is_model_call", isToolCall+" AS is_tool_call")
sb.SelectMore(usage...)
sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
sb.Where(
sb.E("trace_id", traceID),
sb.GE("ts_bucket_start", bounds.Start.Unix()-1800),
sb.LE("ts_bucket_start", bounds.End.Unix()),
isGenAI,
)
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
spans := []spantypes.AISpanUsage{}
if err := s.telemetryStore.ClickhouseDB().Select(ctx, &spans, query, args...); err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error querying gen_ai span usage")
}
return spans, nil
}
func (s *traceStore) GetLinkedSpans(ctx context.Context, traceID string, bounds *spantypes.TraceBounds) ([]spantypes.LinkedSpan, error) {
sb := sqlbuilder.NewSelectBuilder()
sb.Select("DISTINCT ON (span_id) span_id", "links")
sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
sb.Where(
sb.E("trace_id", traceID),
sb.GE("ts_bucket_start", bounds.Start.Unix()-1800),
sb.LE("ts_bucket_start", bounds.End.Unix()),
// The collector writes a link as a FOLLOWS_FROM reference; the parent is a CHILD_OF one.
"position(links, 'FOLLOWS_FROM') > 0",
)
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
var rows []spanLinksRow
if err := s.telemetryStore.ClickhouseDB().Select(ctx, &rows, query, args...); err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error querying linked spans")
}
linkedSpans := make([]spantypes.LinkedSpan, 0, len(rows))
for _, row := range rows {
if linked, ok := spantypes.NewLinkedSpan(row.SpanID, row.Links); ok {
linkedSpans = append(linkedSpans, linked)
}
}
return linkedSpans, nil
}
func (s *traceStore) GetSpanStatusMessages(ctx context.Context, traceID string, bounds *spantypes.TraceBounds, spanIDs []string) (map[string]string, error) {
messages := make(map[string]string, len(spanIDs))
if len(spanIDs) == 0 {
return messages, nil
}
ids := make([]any, len(spanIDs))
for i, id := range spanIDs {
ids[i] = id
}
sb := sqlbuilder.NewSelectBuilder()
sb.Select("DISTINCT ON (span_id) span_id", "status_message")
sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
sb.Where(
sb.E("trace_id", traceID),
sb.In("span_id", ids...),
sb.GE("ts_bucket_start", bounds.Start.Unix()-1800),
sb.LE("ts_bucket_start", bounds.End.Unix()),
)
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
var rows []spanStatusMessageRow
if err := s.telemetryStore.ClickhouseDB().Select(ctx, &rows, query, args...); err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error querying span status messages")
}
for _, row := range rows {
messages[row.SpanID] = row.StatusMessage
}
return messages, nil
}

View File

@@ -21,6 +21,7 @@ import (
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var (
@@ -253,3 +254,33 @@ func TestGetThreadSpans(t *testing.T) {
})
}
}
func TestGetLinkedSpans(t *testing.T) {
columns := []cmock.ColumnType{{Name: "span_id", Type: "String"}, {Name: "links", Type: "String"}}
rows := [][]any{
{"consumer", `[{"traceId":"trace-abc123","spanId":"parent","refType":"CHILD_OF"},{"traceId":"producer-trace","spanId":"producer","refType":"FOLLOWS_FROM"}]`},
{"parent-named-follows-from", `[{"traceId":"trace-abc123","spanId":"FOLLOWS_FROM","refType":"CHILD_OF"}]`},
{"malformed", `not json`},
}
s := newTestStore(sqlmock.QueryMatcherEqual)
s.Mock().ExpectSelect("SELECT DISTINCT ON (span_id) span_id, links FROM signoz_traces.distributed_signoz_index_v3 WHERE trace_id = ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND position(links, 'FOLLOWS_FROM') > 0").
WillReturnRows(cmock.NewRows(columns, rows))
linkedSpans, err := s.Store().GetLinkedSpans(context.Background(), testTraceID, testBounds)
require.NoError(t, err)
assert.Equal(t, []spantypes.LinkedSpan{{SpanID: "consumer", Links: []spantypes.Link{{TraceID: "producer-trace", SpanID: "producer"}}}}, linkedSpans)
assert.NoError(t, s.Mock().ExpectationsWereMet())
}
// The JSON attributes are read as soon as that column is registered, with no feature flag set.
func TestGetAISpanUsageReadsJSONAttributesWithoutFlag(t *testing.T) {
jsonInsideTrace := testStart.Add(500 * time.Second)
s := newTestStoreWithMetadata(sqlmock.QueryMatcherRegexp, genAIMetadataStore(&jsonInsideTrace), nil)
s.Mock().ExpectSelect("attributes\\.`gen_ai\\.request\\.model` IS NOT NULL").WillReturnRows(cmock.NewRows(nil, nil))
_, err := s.Store().GetAISpanUsage(context.Background(), valuer.GenerateUUID(), testTraceID, testBounds)
require.NoError(t, err)
assert.NoError(t, s.Mock().ExpectationsWereMet())
}

View File

@@ -16,6 +16,7 @@ type Handler interface {
GetTraceAggregations(http.ResponseWriter, *http.Request)
GetFlamegraph(http.ResponseWriter, *http.Request)
GetThread(http.ResponseWriter, *http.Request)
InspectTrace(http.ResponseWriter, *http.Request)
}
// Module defines the business logic for trace detail operations.
@@ -25,4 +26,5 @@ type Module interface {
GetTraceAggregations(ctx context.Context, traceID string, req *spantypes.PostableTraceAggregations) (*spantypes.GettableTraceAggregations, error)
GetFlamegraph(ctx context.Context, traceID string, selectedSpanID string, selectFields []telemetrytypes.TelemetryFieldKey) (*spantypes.GettableFlamegraphTrace, error)
GetThread(ctx context.Context, orgID valuer.UUID, traceID string, query *spantypes.ThreadQuery) (*spantypes.GettableTraceThread, error)
InspectTrace(ctx context.Context, orgID valuer.UUID, traceID string, query *spantypes.InspectQuery) (*spantypes.GettableTraceInspection, error)
}

View File

@@ -40,6 +40,12 @@ type TraceStore interface {
GetThreadSpans(ctx context.Context, orgID valuer.UUID, traceID string, bounds *TraceBounds, page ThreadPage) ([]StorableSpan, error)
GetThreadCursor(ctx context.Context, traceID string, bounds *TraceBounds, spanID string) (*ThreadCursor, error)
GetTraceSkeleton(ctx context.Context, traceID string, bounds *TraceBounds) (*TraceSkeleton, error)
GetAISpanUsage(ctx context.Context, orgID valuer.UUID, traceID string, bounds *TraceBounds) ([]AISpanUsage, error)
// GetLinkedSpans returns the spans of the trace that link to other spans.
GetLinkedSpans(ctx context.Context, traceID string, bounds *TraceBounds) ([]LinkedSpan, error)
GetSpanStatusMessages(ctx context.Context, traceID string, bounds *TraceBounds, spanIDs []string) (map[string]string, error)
GetSpanCountByField(ctx context.Context, traceID string, bounds *TraceBounds, fieldKey telemetrytypes.TelemetryFieldKey) (map[string]uint64, error)
GetSpanDurationByField(ctx context.Context, traceID string, bounds *TraceBounds, fieldKey telemetrytypes.TelemetryFieldKey) (map[string]uint64, error)
}

View File

@@ -0,0 +1,336 @@
package spantypes
import (
"encoding/base64"
"encoding/json"
"slices"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/valuer"
)
const (
inspectListSize = 10
inspectServiceLimit = 20
inspectLinkTargets = 5
// inspectTreeSpans is the number of spans the tree shows, and the largest trace returned with its records.
inspectTreeSpans = 50
inspectSmallTraceChars = 20_000
// inspectMaxResponseChars keeps a response under 50,000 characters with its envelope.
inspectMaxResponseChars = 48_000
)
var (
ErrCodeInspectInvalidCursor = errors.MustNewCode("trace_inspect_invalid_cursor")
ErrCodeInspectInvalidDepth = errors.MustNewCode("trace_inspect_invalid_depth")
ErrCodeInspectInvalidLimit = errors.MustNewCode("trace_inspect_invalid_limit")
ErrCodeInspectInvalidParams = errors.MustNewCode("trace_inspect_invalid_params")
)
// The lists of the overview, as a cursor names them.
const (
OverviewListErrors = "errors"
OverviewListUnmarkedFailures = "unmarkedFailures"
OverviewListSlowest = "slowest"
OverviewListExpensive = "expensive"
OverviewListLinks = "links"
)
var overviewLists = []string{OverviewListErrors, OverviewListUnmarkedFailures, OverviewListSlowest, OverviewListExpensive, OverviewListLinks}
var (
UnmarkedFailureHTTP4xx = UnmarkedFailureReason{valuer.NewString("http_4xx")}
UnmarkedFailureGRPCStatus = UnmarkedFailureReason{valuer.NewString("grpc_status")}
UnmarkedFailureExceptionEvent = UnmarkedFailureReason{valuer.NewString("exception_event")}
)
var (
InspectTypeOverview = InspectType{valuer.NewString("overview")}
InspectTypeSpan = InspectType{valuer.NewString("span")}
InspectTypeFind = InspectType{valuer.NewString("find")}
)
// UnmarkedFailureReason says why a span without the error flag is listed as a failure.
type UnmarkedFailureReason struct{ valuer.String }
func (UnmarkedFailureReason) Enum() []any {
return []any{UnmarkedFailureHTTP4xx, UnmarkedFailureGRPCStatus, UnmarkedFailureExceptionEvent}
}
// InspectType names the part of an inspect response that is filled.
type InspectType struct{ valuer.String }
func (InspectType) Enum() []any {
return []any{InspectTypeOverview, InspectTypeSpan, InspectTypeFind}
}
type InspectTraceParams struct {
SpanID string `query:"spanId" description:"Returns this span in full with the spans around it. Cannot be combined with search."`
Search string `query:"search" description:"Returns the spans that match, earliest first: words, a filter expression, or both, such as worldpay AND has_error = true. Words match the span name, status message, attribute values and events, ignoring case."`
Depth int `query:"depth" description:"With spanId, the number of levels of ancestors and of descendants returned, 1 to 10; 2 when not given. The root span is always returned."`
AllSpans bool `query:"allSpans" description:"Returns a different response: every span of the trace as a short line, in tree order, or with spanId that span and every span below it. Cannot be combined with search or depth."`
SelectFields []string `query:"selectFields" description:"With spanId, attribute names returned without the 500 character cut; the record then holds only these attributes."`
Limit int `query:"limit" description:"Page size for search and for allSpans, at most 1000; when not given, 20 with search and 400 with allSpans. A page holds fewer spans when it would pass the 50,000 character limit."`
Cursor string `query:"cursor" description:"The nextCursor of an earlier response to the same request; returns the next page."`
}
// SpanSummary is the short form of a span: enough to place it in the trace and decide whether to open it.
type SpanSummary struct {
SpanID string `json:"spanId" required:"true"`
ParentSpanID string `json:"parentSpanId,omitempty"`
Name string `json:"name" required:"true"`
ServiceName string `json:"serviceName" required:"true"`
Kind string `json:"kind" required:"true"`
// StartOffsetNano is the time from the start of the trace to the start of the span.
StartOffsetNano uint64 `json:"startOffsetNano" required:"true"`
DurationNano uint64 `json:"durationNano" required:"true"`
// OwnTimeNano is the duration minus the time the span's children cover.
OwnTimeNano uint64 `json:"ownTimeNano" required:"true"`
ResponseStatusCode string `json:"responseStatusCode,omitempty"`
HasError bool `json:"hasError" required:"true"`
Level uint32 `json:"level" required:"true"`
ChildCount uint32 `json:"childCount" required:"true"`
// SubtreeSpanCount is the number of spans below this one.
SubtreeSpanCount uint32 `json:"subtreeSpanCount" required:"true"`
}
type TraceServiceStats struct {
ServiceName string `json:"serviceName" required:"true"`
SpanCount uint64 `json:"spanCount" required:"true"`
ErrorSpanCount uint64 `json:"errorSpanCount" required:"true"`
OwnTimeNano uint64 `json:"ownTimeNano" required:"true"`
}
type TraceAIIdentity struct {
ModelCallCount uint64 `json:"modelCallCount" required:"true"`
ToolCallCount uint64 `json:"toolCallCount" required:"true"`
Tokens TraceAITokens `json:"tokens" required:"true"`
// TotalCost is omitted when no span carries a cost attribute.
TotalCost *float64 `json:"totalCost,omitempty" nullable:"false"`
}
type TraceIdentity struct {
TraceID string `json:"traceId" required:"true"`
StartTimestampMillis uint64 `json:"startTimestampMillis" required:"true"`
DurationNano uint64 `json:"durationNano" required:"true"`
// Root is the earliest span without a parent; omitted when every span's parent is missing.
Root *SpanSummary `json:"root,omitempty"`
SpanCount uint64 `json:"spanCount" required:"true"`
ErrorSpanCount uint64 `json:"errorSpanCount" required:"true"`
RootCount uint64 `json:"rootCount" required:"true"`
HasMissingSpans bool `json:"hasMissingSpans" required:"true"`
ServiceCount uint64 `json:"serviceCount" required:"true"`
Services []TraceServiceStats `json:"services" required:"true" nullable:"false"`
AI *TraceAIIdentity `json:"ai,omitempty"`
}
// Every list has a total, items and a nextCursor when more items follow. Every item starts with
// its span and that span's direct parent, omitted for a root or when the parent span is missing.
type TraceErrorItem struct {
Span SpanSummary `json:"span" required:"true"`
Parent *SpanSummary `json:"parent,omitempty"`
StatusMessage string `json:"statusMessage" required:"true"`
// ErrorAncestorCount is the number of consecutive ancestors, from the parent up, that carry the error flag.
ErrorAncestorCount uint32 `json:"errorAncestorCount" required:"true"`
}
type TraceErrorList struct {
Total uint64 `json:"total" required:"true"`
Items []TraceErrorItem `json:"items" required:"true" nullable:"false"`
NextCursor string `json:"nextCursor,omitempty"`
}
type TraceUnmarkedFailureItem struct {
Span SpanSummary `json:"span" required:"true"`
Parent *SpanSummary `json:"parent,omitempty"`
Reason UnmarkedFailureReason `json:"reason" required:"true"`
}
type TraceUnmarkedFailureList struct {
Total uint64 `json:"total" required:"true"`
Items []TraceUnmarkedFailureItem `json:"items" required:"true" nullable:"false"`
NextCursor string `json:"nextCursor,omitempty"`
}
// TraceSlowestItem is one span, or the same-name spans under one parent taken together.
type TraceSlowestItem struct {
// Span is the longest span of the group.
Span SpanSummary `json:"span" required:"true"`
Parent *SpanSummary `json:"parent,omitempty"`
SpanCount uint64 `json:"spanCount" required:"true"`
OwnTimeNano uint64 `json:"ownTimeNano" required:"true"`
ErrorSpanCount uint64 `json:"errorSpanCount" required:"true"`
}
type TraceSlowestList struct {
Total uint64 `json:"total" required:"true"`
Items []TraceSlowestItem `json:"items" required:"true" nullable:"false"`
NextCursor string `json:"nextCursor,omitempty"`
}
type TraceExpensiveItem struct {
Span SpanSummary `json:"span" required:"true"`
Parent *SpanSummary `json:"parent,omitempty"`
Cost *float64 `json:"cost,omitempty" nullable:"false"`
InputTokens uint64 `json:"inputTokens" required:"true"`
OutputTokens uint64 `json:"outputTokens" required:"true"`
}
type TraceExpensiveList struct {
Total uint64 `json:"total" required:"true"`
Items []TraceExpensiveItem `json:"items" required:"true" nullable:"false"`
NextCursor string `json:"nextCursor,omitempty"`
}
type TraceLinkItem struct {
Span SpanSummary `json:"span" required:"true"`
Parent *SpanSummary `json:"parent,omitempty"`
// LinkCount is the number of spans this span links to; Links holds the first of them.
LinkCount uint64 `json:"linkCount" required:"true"`
Links []Link `json:"links" required:"true" nullable:"false"`
}
// TraceLinkList lists the spans of this trace that link to other spans, in this or another trace.
type TraceLinkList struct {
Total uint64 `json:"total" required:"true"`
Items []TraceLinkItem `json:"items" required:"true" nullable:"false"`
NextCursor string `json:"nextCursor,omitempty"`
}
type TraceTree struct {
// Total is the number of spans in the trace; Spans holds every one of them when Complete is set.
Total uint64 `json:"total" required:"true"`
Complete bool `json:"complete" required:"true"`
// Spans is in tree order: a span is followed by the spans below it. A span's childCount and
// subtreeSpanCount say how much of the trace sits under it.
Spans []SpanSummary `json:"spans" required:"true" nullable:"false"`
// Records holds the full record of every span of a small trace.
Records []SpanRecord `json:"records,omitempty" nullable:"false"`
}
// TraceOverview holds every part on a first call and one list on a call with a cursor.
type TraceOverview struct {
Identity *TraceIdentity `json:"identity,omitempty"`
Errors *TraceErrorList `json:"errors,omitempty"`
UnmarkedFailures *TraceUnmarkedFailureList `json:"unmarkedFailures,omitempty"`
Slowest *TraceSlowestList `json:"slowest,omitempty"`
// Expensive is present when the trace has gen_ai spans.
Expensive *TraceExpensiveList `json:"expensive,omitempty"`
Links *TraceLinkList `json:"links,omitempty"`
Tree *TraceTree `json:"tree,omitempty"`
}
type TraceSpanLevel struct {
// Level counts down from the span: 1 holds its children.
Level uint32 `json:"level" required:"true"`
Total uint64 `json:"total" required:"true"`
Spans []SpanSummary `json:"spans" required:"true" nullable:"false"`
}
// TraceSpanPercentile places a span's duration among the spans of the same name and service that
// started in the window before it.
type TraceSpanPercentile struct {
P50Nano uint64 `json:"p50Nano" required:"true"`
P90Nano uint64 `json:"p90Nano" required:"true"`
P99Nano uint64 `json:"p99Nano" required:"true"`
// Position is the share of those spans, 0 to 100, that were no slower than this one.
Position float64 `json:"position" required:"true"`
WindowMinutes uint64 `json:"windowMinutes" required:"true"`
}
type TraceSpanDetail struct {
Summary SpanSummary `json:"summary" required:"true"`
Record SpanRecord `json:"record" required:"true"`
// Percentile is omitted when it could not be computed in time.
Percentile *TraceSpanPercentile `json:"percentile,omitempty"`
// Ancestors holds the nearest ancestors, the parent first, then the root when it is further up.
Ancestors []SpanSummary `json:"ancestors" required:"true" nullable:"false"`
AncestorCount uint64 `json:"ancestorCount" required:"true"`
Descendants []TraceSpanLevel `json:"descendants" required:"true" nullable:"false"`
}
type SpanTextMatch struct {
Field string `json:"field" required:"true"`
Snippet string `json:"snippet" required:"true"`
}
type TraceSpanMatch struct {
Span SpanSummary `json:"span" required:"true"`
Parent *SpanSummary `json:"parent,omitempty"`
// Matches says where the searched words were found; empty for a filter without words.
Matches []SpanTextMatch `json:"matches" required:"true" nullable:"false"`
}
type TraceSpanList struct {
Total uint64 `json:"total" required:"true"`
Items []TraceSpanMatch `json:"items" required:"true" nullable:"false"`
NextCursor string `json:"nextCursor,omitempty"`
}
// GettableTraceInspection holds one part, named by Type.
type GettableTraceInspection struct {
Type InspectType `json:"type" required:"true"`
Overview *TraceOverview `json:"overview,omitempty"`
Span *TraceSpanDetail `json:"span,omitempty"`
// Find holds the spans matching search, or every span for allSpans.
Find *TraceSpanList `json:"find,omitempty"`
}
// InspectQuery is a validated inspect request.
type InspectQuery struct {
Type InspectType
// List and Offset page one list of the overview; List is empty on a first call.
List string
Offset int
}
// inspectCursor is the content of a nextCursor: the list it belongs to and where its next page starts.
type inspectCursor struct {
List string `json:"list,omitempty"`
Offset int `json:"offset,omitempty"`
}
// NewInspectQuery validates the parameters of an overview request. A parameter that does not
// apply to the overview is an error.
func NewInspectQuery(params *InspectTraceParams) (*InspectQuery, error) {
query := &InspectQuery{Type: InspectTypeOverview}
switch {
case params.SpanID != "" || params.Search != "" || params.AllSpans:
return nil, errors.New(errors.TypeUnsupported, errors.CodeUnsupported, "spanId, search and allSpans are not implemented yet")
case params.Depth != 0:
return nil, errors.NewInvalidInputf(ErrCodeInspectInvalidDepth, "depth needs spanId")
case len(params.SelectFields) > 0:
return nil, errors.NewInvalidInputf(ErrCodeInspectInvalidParams, "selectFields needs spanId")
case params.Limit != 0:
return nil, errors.NewInvalidInputf(ErrCodeInspectInvalidLimit, "limit needs search or allSpans")
case params.Cursor == "":
return query, nil
}
cursor, err := decodeInspectCursor(params.Cursor)
if err != nil {
return nil, err
}
if !slices.Contains(overviewLists, cursor.List) || cursor.Offset <= 0 {
return nil, errors.NewInvalidInputf(ErrCodeInspectInvalidCursor, "invalid cursor")
}
query.List, query.Offset = cursor.List, cursor.Offset
return query, nil
}
func decodeInspectCursor(encoded string) (*inspectCursor, error) {
data, err := base64.RawURLEncoding.DecodeString(encoded)
if err != nil {
return nil, errors.WrapInvalidInputf(err, ErrCodeInspectInvalidCursor, "invalid cursor")
}
cursor := new(inspectCursor)
if err := json.Unmarshal(data, cursor); err != nil {
return nil, errors.WrapInvalidInputf(err, ErrCodeInspectInvalidCursor, "invalid cursor")
}
return cursor, nil
}
func (c inspectCursor) encode() string {
data, _ := json.Marshal(c)
return base64.RawURLEncoding.EncodeToString(data)
}

View File

@@ -0,0 +1,66 @@
package spantypes
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/SigNoz/signoz/pkg/errors"
)
func TestNewInspectQuery(t *testing.T) {
validCases := []struct {
name string
params InspectTraceParams
expected InspectQuery
}{
{name: "NoParams_Overview", params: InspectTraceParams{}, expected: InspectQuery{Type: InspectTypeOverview}},
{name: "ErrorsCursor_PageOfErrors", params: InspectTraceParams{Cursor: inspectCursor{List: OverviewListErrors, Offset: 10}.encode()}, expected: InspectQuery{Type: InspectTypeOverview, List: OverviewListErrors, Offset: 10}},
{name: "SlowestCursor_PageOfSlowest", params: InspectTraceParams{Cursor: inspectCursor{List: OverviewListSlowest, Offset: 20}.encode()}, expected: InspectQuery{Type: InspectTypeOverview, List: OverviewListSlowest, Offset: 20}},
}
for _, testCase := range validCases {
t.Run(testCase.name, func(t *testing.T) {
query, err := NewInspectQuery(&testCase.params)
require.NoError(t, err)
assert.Equal(t, testCase.expected, *query)
})
}
invalidCases := []struct {
name string
params InspectTraceParams
}{
{name: "Depth_NeedsSpanID", params: InspectTraceParams{Depth: 3}},
{name: "SelectFields_NeedsSpanID", params: InspectTraceParams{SelectFields: []string{"db.statement"}}},
{name: "Limit_NeedsSearchOrAllSpans", params: InspectTraceParams{Limit: 10}},
{name: "Cursor_NotBase64", params: InspectTraceParams{Cursor: "%%%"}},
{name: "Cursor_NotJSON", params: InspectTraceParams{Cursor: "bm90IGpzb24"}},
{name: "Cursor_UnknownList", params: InspectTraceParams{Cursor: inspectCursor{List: "tree", Offset: 10}.encode()}},
{name: "Cursor_WithoutList", params: InspectTraceParams{Cursor: inspectCursor{Offset: 10}.encode()}},
{name: "Cursor_WithoutOffset", params: InspectTraceParams{Cursor: inspectCursor{List: OverviewListErrors}.encode()}},
}
for _, testCase := range invalidCases {
t.Run(testCase.name, func(t *testing.T) {
_, err := NewInspectQuery(&testCase.params)
require.Error(t, err)
assert.True(t, errors.Ast(err, errors.TypeInvalidInput))
})
}
notImplementedCases := []struct {
name string
params InspectTraceParams
}{
{name: "SpanID", params: InspectTraceParams{SpanID: "abc"}},
{name: "Search", params: InspectTraceParams{Search: "worldpay"}},
{name: "AllSpans", params: InspectTraceParams{AllSpans: true}},
}
for _, testCase := range notImplementedCases {
t.Run(testCase.name+"_NotImplementedYet", func(t *testing.T) {
_, err := NewInspectQuery(&testCase.params)
require.Error(t, err)
assert.True(t, errors.Ast(err, errors.TypeUnsupported))
})
}
}

View File

@@ -0,0 +1,422 @@
package spantypes
import (
"container/heap"
"encoding/json"
"slices"
"strconv"
)
// errorOrigins lists the error spans with no error span among their children, in start time order.
func (s *TraceSkeleton) errorOrigins() []int32 {
origins := make([]int32, 0)
for i := range int32(s.Len()) {
if !s.HasError(i) {
continue
}
hasErrorChild := false
for _, child := range s.Children(i) {
if s.HasError(child) {
hasErrorChild = true
break
}
}
if !hasErrorChild {
origins = append(origins, i)
}
}
return origins
}
// ErrorPage returns one page of error origins and the ids of its spans, for their status messages.
func (s *TraceSkeleton) ErrorPage(offset int) (*TraceErrorList, []string) {
origins := s.errorOrigins()
list := &TraceErrorList{Total: uint64(len(origins)), Items: []TraceErrorItem{}, NextCursor: nextListCursor(OverviewListErrors, offset, len(origins))}
spanIDs := make([]string, 0, inspectListSize)
for _, i := range pageOf(origins, offset, inspectListSize) {
item := TraceErrorItem{Span: s.Summary(i), Parent: s.parentSummary(i)}
for p := s.Parents[i]; p != noParent && s.HasError(p); p = s.Parents[p] {
item.ErrorAncestorCount++
}
list.Items = append(list.Items, item)
spanIDs = append(spanIDs, item.Span.SpanID)
}
return list, spanIDs
}
// SetStatusMessages fills the status message of each item, cut like any other long value.
func (list *TraceErrorList) SetStatusMessages(messages map[string]string) {
for i := range list.Items {
message := messages[list.Items[i].Span.SpanID]
if cut, _, ok := cutString(message, spanRecordValueChars); ok {
message = cut
}
list.Items[i].StatusMessage = message
}
}
// UnmarkedFailurePage returns one page of the spans that failed without the error flag: a 4xx
// response, a gRPC status other than OK, or an exception event.
func (s *TraceSkeleton) UnmarkedFailurePage(offset int) *TraceUnmarkedFailureList {
type failure struct {
index int32
reason UnmarkedFailureReason
}
failures := make([]failure, 0)
for i := range int32(s.Len()) {
if s.HasError(i) {
continue
}
if reason, ok := unmarkedFailureReason(s.ResponseStatusCode(i), s.HasExceptionEvent(i)); ok {
failures = append(failures, failure{index: i, reason: reason})
}
}
list := &TraceUnmarkedFailureList{Total: uint64(len(failures)), Items: []TraceUnmarkedFailureItem{}, NextCursor: nextListCursor(OverviewListUnmarkedFailures, offset, len(failures))}
for _, f := range pageOf(failures, offset, inspectListSize) {
list.Items = append(list.Items, TraceUnmarkedFailureItem{Span: s.Summary(f.index), Parent: s.parentSummary(f.index), Reason: f.reason})
}
return list
}
// unmarkedFailureReason reads the response status column, which holds an HTTP status or a gRPC status.
func unmarkedFailureReason(responseStatusCode string, hasExceptionEvent bool) (UnmarkedFailureReason, bool) {
if code, err := strconv.Atoi(responseStatusCode); err == nil {
switch {
case code >= 400 && code < 500:
return UnmarkedFailureHTTP4xx, true
case code >= 1 && code <= 16:
return UnmarkedFailureGRPCStatus, true
}
}
if hasExceptionEvent {
return UnmarkedFailureExceptionEvent, true
}
return UnmarkedFailureReason{}, false
}
// LinkPage returns one page of the spans that link to other spans, in start time order.
func (s *TraceSkeleton) LinkPage(offset int) *TraceLinkList {
list := &TraceLinkList{Total: uint64(len(s.LinkedSpans)), Items: []TraceLinkItem{}, NextCursor: nextListCursor(OverviewListLinks, offset, len(s.LinkedSpans))}
for _, linked := range pageOf(s.LinkedSpans, offset, inspectListSize) {
idx, ok := s.Index(linked.SpanID)
if !ok {
continue
}
list.Items = append(list.Items, TraceLinkItem{
Span: s.Summary(idx),
Parent: s.parentSummary(idx),
LinkCount: uint64(len(linked.Links)),
Links: linked.Links[:min(len(linked.Links), inspectLinkTargets)],
})
}
return list
}
// slowestEntry is one span, or the same-name spans under one parent taken together.
type slowestEntry struct {
longest int32
spanCount uint64
ownTime uint64
errCount uint64
}
// ranksBefore orders entries by own time, the largest first, and by span order between equals,
// so that every page cuts the same ranking.
func (e slowestEntry) ranksBefore(other slowestEntry) bool {
if e.ownTime != other.ownTime {
return e.ownTime > other.ownTime
}
return e.longest < other.longest
}
// slowestHeap holds the best-ranked entries seen so far, the last of them on top.
type slowestHeap []slowestEntry
func (h slowestHeap) Len() int { return len(h) }
func (h slowestHeap) Less(i, j int) bool { return h[j].ranksBefore(h[i]) }
func (h slowestHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h *slowestHeap) Push(x any) { *h = append(*h, x.(slowestEntry)) }
func (h *slowestHeap) Pop() any {
old := *h
last := old[len(old)-1]
*h = old[:len(old)-1]
return last
}
// SlowestPage returns one page of the ranking by own time, same-name spans under one parent
// counted as one entry. Only the entries up to the end of the page are kept while ranking.
func (s *TraceSkeleton) SlowestPage(offset int) *TraceSlowestList {
keep := offset + inspectListSize
top := &slowestHeap{}
total := 0
offer := func(e slowestEntry) {
total++
switch {
case top.Len() < keep:
heap.Push(top, e)
case e.ranksBefore((*top)[0]):
(*top)[0] = e
heap.Fix(top, 0)
}
}
single := func(i int32) slowestEntry {
e := slowestEntry{longest: i, spanCount: 1, ownTime: s.ownTime[i]}
if s.HasError(i) {
e.errCount = 1
}
return e
}
for _, root := range s.roots {
offer(single(root))
}
var siblings []int32
for parent := range int32(s.Len()) {
children := s.Children(parent)
if len(children) == 1 {
offer(single(children[0]))
continue
}
siblings = append(siblings[:0], children...)
slices.SortStableFunc(siblings, func(a, b int32) int {
if s.Names[a] != s.Names[b] {
return int(s.Names[a]) - int(s.Names[b])
}
return int(s.Services[a]) - int(s.Services[b])
})
for k := 0; k < len(siblings); {
e := single(siblings[k])
next := k + 1
for ; next < len(siblings) && s.Names[siblings[next]] == s.Names[siblings[k]] && s.Services[siblings[next]] == s.Services[siblings[k]]; next++ {
member := single(siblings[next])
e.spanCount++
e.ownTime += member.ownTime
e.errCount += member.errCount
if s.DurationNano[siblings[next]] > s.DurationNano[e.longest] {
e.longest = siblings[next]
}
}
offer(e)
k = next
}
}
ranked := []slowestEntry(*top)
slices.SortFunc(ranked, func(a, b slowestEntry) int {
if a.ranksBefore(b) {
return -1
}
return 1
})
list := &TraceSlowestList{Total: uint64(total), Items: []TraceSlowestItem{}, NextCursor: nextListCursor(OverviewListSlowest, offset, total)}
for _, e := range pageOf(ranked, offset, inspectListSize) {
list.Items = append(list.Items, TraceSlowestItem{Span: s.Summary(e.longest), Parent: s.parentSummary(e.longest), SpanCount: e.spanCount, OwnTimeNano: e.ownTime, ErrorSpanCount: e.errCount})
}
return list
}
// ExpensivePage returns one page of the gen_ai spans ranked by cost, or by tokens when no span
// carries a cost; nil without gen_ai spans.
func (s *TraceSkeleton) ExpensivePage(offset int) *TraceExpensiveList {
if len(s.AISpans) == 0 {
return nil
}
hasCost := false
for i := range s.AISpans {
if s.AISpans[i].Cost != nil {
hasCost = true
break
}
}
type rankedSpan struct {
index int32
weight float64
usage *AISpanUsage
}
ranked := make([]rankedSpan, 0, len(s.AISpans))
for i := range s.AISpans {
usage := &s.AISpans[i]
index, ok := s.Index(usage.SpanID)
if !ok {
continue
}
weight := floatValue(usage.InputTokens) + floatValue(usage.OutputTokens)
if hasCost {
weight = floatValue(usage.Cost)
}
ranked = append(ranked, rankedSpan{index: index, weight: weight, usage: usage})
}
// Equal weights fall back to span order, so that every page cuts the same ranking.
slices.SortFunc(ranked, func(a, b rankedSpan) int {
switch {
case a.weight > b.weight:
return -1
case a.weight < b.weight:
return 1
}
return int(a.index) - int(b.index)
})
list := &TraceExpensiveList{Total: uint64(len(ranked)), Items: []TraceExpensiveItem{}, NextCursor: nextListCursor(OverviewListExpensive, offset, len(ranked))}
for _, span := range pageOf(ranked, offset, inspectListSize) {
list.Items = append(list.Items, TraceExpensiveItem{
Span: s.Summary(span.index),
Parent: s.parentSummary(span.index),
Cost: span.usage.Cost,
InputTokens: uint64(floatValue(span.usage.InputTokens)),
OutputTokens: uint64(floatValue(span.usage.OutputTokens)),
})
}
return list
}
func (s *TraceSkeleton) Identity() *TraceIdentity {
identity := &TraceIdentity{
TraceID: s.TraceID,
StartTimestampMillis: uint64(s.startNs / 1_000_000),
DurationNano: uint64(s.endNs - s.startNs),
SpanCount: uint64(s.Len()),
ErrorSpanCount: s.errCount,
RootCount: uint64(len(s.roots)),
HasMissingSpans: s.HasMissingSpans(),
}
if root, ok := s.PrimaryRoot(); ok && !s.HasMissingParent(root) {
summary := s.Summary(root)
identity.Root = &summary
}
stats := make([]TraceServiceStats, len(s.Strings))
for i := range int32(s.Len()) {
service := &stats[s.Services[i]]
service.SpanCount++
service.OwnTimeNano += s.ownTime[i]
if s.HasError(i) {
service.ErrorSpanCount++
}
}
services := make([]TraceServiceStats, 0)
for idx := range stats {
if stats[idx].SpanCount > 0 {
stats[idx].ServiceName = s.Strings[idx]
services = append(services, stats[idx])
}
}
slices.SortStableFunc(services, func(a, b TraceServiceStats) int {
switch {
case a.OwnTimeNano > b.OwnTimeNano:
return -1
case a.OwnTimeNano < b.OwnTimeNano:
return 1
}
return 0
})
identity.ServiceCount = uint64(len(services))
identity.Services = services[:min(len(services), inspectServiceLimit)]
if len(s.AISpans) > 0 {
ai := &TraceAIIdentity{}
for i := range s.AISpans {
usage := &s.AISpans[i]
if usage.IsModelCall {
ai.ModelCallCount++
}
if usage.IsToolCall {
ai.ToolCallCount++
}
ai.Tokens.Input += uint64(floatValue(usage.InputTokens))
ai.Tokens.Output += uint64(floatValue(usage.OutputTokens))
ai.Tokens.CacheRead += uint64(floatValue(usage.CacheReadTokens))
ai.Tokens.CacheWrite += uint64(floatValue(usage.CacheWriteTokens))
ai.Tokens.Reasoning += uint64(floatValue(usage.ReasoningTokens))
if usage.Cost != nil {
if ai.TotalCost == nil {
ai.TotalCost = new(float64)
}
*ai.TotalCost += *usage.Cost
}
}
identity.AI = ai
}
return identity
}
// Tree lists the first 50 spans of the trace: level by level from the roots, and in time order on
// the level that does not fit. It is complete when the trace has no more spans than that.
func (s *TraceSkeleton) Tree() *TraceTree {
return s.tree(inspectTreeSpans)
}
func (s *TraceSkeleton) tree(limit int) *TraceTree {
tree := &TraceTree{Total: uint64(s.Len()), Complete: s.Len() <= limit}
shown := make(map[int32]struct{}, min(limit, s.Len()))
for level := s.roots; len(level) > 0 && len(shown) < limit; {
level = level[:min(len(level), limit-len(shown))]
next := make([]int32, 0)
for _, i := range level {
shown[i] = struct{}{}
next = append(next, s.Children(i)...)
}
level = next
}
// The shown spans are returned in tree order: a span is followed by the spans below it.
tree.Spans = make([]SpanSummary, 0, len(shown))
for _, i := range s.preorder {
if _, ok := shown[i]; ok {
tree.Spans = append(tree.Spans, s.Summary(i))
}
}
return tree
}
// FitOverview drops the span records and then shrinks the tree until the overview fits the
// response size limit.
func (s *TraceSkeleton) FitOverview(overview *TraceOverview) {
for limit := len(overview.Tree.Spans); jsonChars(overview) > inspectMaxResponseChars && limit > 0; {
if overview.Tree.Records != nil {
overview.Tree.Records = nil
continue
}
over := jsonChars(overview) - inspectMaxResponseChars
perSpan := jsonChars(overview.Tree.Spans)/len(overview.Tree.Spans) + 1
limit = max(limit-over/perSpan-1, 0)
overview.Tree = s.tree(limit)
}
}
// parentSummary returns the short form of span i's direct parent; nil for a root or a missing parent.
func (s *TraceSkeleton) parentSummary(i int32) *SpanSummary {
parent := s.Parents[i]
if parent == noParent {
return nil
}
summary := s.Summary(parent)
return &summary
}
// nextListCursor returns the cursor of the page after the one starting at offset; empty on the last page.
func nextListCursor(list string, offset, total int) string {
if offset+inspectListSize >= total {
return ""
}
return inspectCursor{List: list, Offset: offset + inspectListSize}.encode()
}
func pageOf[T any](items []T, offset, limit int) []T {
if offset >= len(items) {
return nil
}
return items[offset:min(offset+limit, len(items))]
}
func jsonChars(value any) int {
data, _ := json.Marshal(value)
return len(data)
}
func floatValue(value *float64) float64 {
if value == nil {
return 0
}
return *value
}

View File

@@ -0,0 +1,484 @@
package spantypes
import (
"fmt"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func summaryIDs(summaries []SpanSummary) []string {
ids := make([]string, 0, len(summaries))
for _, summary := range summaries {
ids = append(ids, summary.SpanID)
}
return ids
}
// nextOffset reads the offset out of a list's nextCursor, as a request carrying it would.
func nextOffset(t *testing.T, list, cursor string) int {
t.Helper()
query, err := NewInspectQuery(&InspectTraceParams{Cursor: cursor})
require.NoError(t, err)
require.Equal(t, list, query.List)
return query.Offset
}
func TestErrorPage(t *testing.T) {
t.Run("ErrorCarriedUpToRoot_ListsWhereItStarted", func(t *testing.T) {
specs := []skeletonSpec{{id: "level-0", startMs: 0, durationMs: 100, hasError: true}}
for depth := 1; depth <= 12; depth++ {
specs = append(specs, skeletonSpec{id: fmt.Sprintf("level-%d", depth), parent: fmt.Sprintf("level-%d", depth-1), startMs: int64(depth), durationMs: 50, hasError: true})
}
specs = append(specs, skeletonSpec{id: "healthy", parent: "level-0", startMs: 60, durationMs: 5})
skeleton := buildSkeleton(specs...)
list, spanIDs := skeleton.ErrorPage(0)
assert.Equal(t, uint64(1), list.Total)
assert.Equal(t, []string{"level-12"}, spanIDs)
require.Len(t, list.Items, 1)
assert.Equal(t, uint32(12), list.Items[0].ErrorAncestorCount)
require.NotNil(t, list.Items[0].Parent)
assert.Equal(t, "level-11", list.Items[0].Parent.SpanID)
assert.Empty(t, list.NextCursor)
})
t.Run("ParentFailsOnItsOwn_ListsChildOnly", func(t *testing.T) {
skeleton := buildSkeleton(
skeletonSpec{id: "root", startMs: 0, durationMs: 100},
skeletonSpec{id: "parent", parent: "root", startMs: 1, durationMs: 50, hasError: true},
skeletonSpec{id: "child", parent: "parent", startMs: 2, durationMs: 10, hasError: true},
skeletonSpec{id: "other", parent: "root", startMs: 60, durationMs: 10, hasError: true},
)
list, spanIDs := skeleton.ErrorPage(0)
assert.Equal(t, uint64(2), list.Total)
assert.Equal(t, []string{"child", "other"}, spanIDs)
})
t.Run("MoreThanOnePage_PagesByCursorInStartOrder", func(t *testing.T) {
specs := []skeletonSpec{{id: "root", startMs: 0, durationMs: 1000}}
for i := range 23 {
specs = append(specs, skeletonSpec{id: fmt.Sprintf("err-%02d", i), parent: "root", startMs: int64(i + 1), durationMs: 1, hasError: true})
}
skeleton := buildSkeleton(specs...)
seen := make([]string, 0, 23)
offset := 0
for range 3 {
list, spanIDs := skeleton.ErrorPage(offset)
assert.Equal(t, uint64(23), list.Total)
seen = append(seen, spanIDs...)
if list.NextCursor == "" {
break
}
offset = nextOffset(t, OverviewListErrors, list.NextCursor)
}
require.Len(t, seen, 23)
assert.Equal(t, "err-00", seen[0])
assert.Equal(t, "err-10", seen[10])
assert.Equal(t, "err-22", seen[22])
})
t.Run("RootError_HasNoParentAndItsMessageIsCut", func(t *testing.T) {
skeleton := buildSkeleton(skeletonSpec{id: "root", startMs: 0, durationMs: 10, hasError: true})
list, _ := skeleton.ErrorPage(0)
message := ""
for range 60 {
message += "0123456789"
}
list.SetStatusMessages(map[string]string{"root": message})
assert.Nil(t, list.Items[0].Parent)
assert.Len(t, []rune(list.Items[0].StatusMessage), spanRecordValueChars+1)
})
}
func TestUnmarkedFailurePage(t *testing.T) {
skeleton := buildSkeleton(
skeletonSpec{id: "root", startMs: 0, durationMs: 100, statusCode: "200"},
skeletonSpec{id: "not-found", parent: "root", startMs: 1, durationMs: 1, statusCode: "404"},
skeletonSpec{id: "flagged-4xx", parent: "root", startMs: 2, durationMs: 1, statusCode: "409", hasError: true},
skeletonSpec{id: "grpc-ok", parent: "root", startMs: 3, durationMs: 1, statusCode: "0"},
skeletonSpec{id: "grpc-not-found", parent: "root", startMs: 4, durationMs: 1, statusCode: "5"},
skeletonSpec{id: "handled-exception", parent: "root", startMs: 5, durationMs: 1, hasException: true},
skeletonSpec{id: "server-error", parent: "root", startMs: 6, durationMs: 1, statusCode: "503"},
skeletonSpec{id: "json-rpc", parent: "root", startMs: 7, durationMs: 1, statusCode: "-32601"},
)
list := skeleton.UnmarkedFailurePage(0)
require.Len(t, list.Items, 3)
assert.Equal(t, uint64(3), list.Total)
assert.Equal(t, "not-found", list.Items[0].Span.SpanID)
assert.Equal(t, UnmarkedFailureHTTP4xx, list.Items[0].Reason)
require.NotNil(t, list.Items[0].Parent)
assert.Equal(t, "root", list.Items[0].Parent.SpanID)
assert.Equal(t, "grpc-not-found", list.Items[1].Span.SpanID)
assert.Equal(t, UnmarkedFailureGRPCStatus, list.Items[1].Reason)
assert.Equal(t, "handled-exception", list.Items[2].Span.SpanID)
assert.Equal(t, UnmarkedFailureExceptionEvent, list.Items[2].Reason)
}
func TestLinkPage(t *testing.T) {
specs := []skeletonSpec{{id: "root", startMs: 0, durationMs: 1000}}
linkedSpans := make([]LinkedSpan, 0, 14)
for i := range 13 {
id := fmt.Sprintf("consumer-%02d", i)
specs = append(specs, skeletonSpec{id: id, parent: "root", startMs: int64(i + 1), durationMs: 1})
// Given in reverse so the page order has to come from the spans, not from the input.
linkedSpans = append([]LinkedSpan{{SpanID: id, Links: []Link{{TraceID: "producer-trace", SpanID: id + "-producer"}}}}, linkedSpans...)
}
batch := LinkedSpan{SpanID: "consumer-00"}
for i := range 8 {
batch.Links = append(batch.Links, Link{TraceID: fmt.Sprintf("producer-trace-%d", i), SpanID: "producer"})
}
linkedSpans[len(linkedSpans)-1] = batch
linkedSpans = append(linkedSpans, LinkedSpan{SpanID: "not-loaded", Links: []Link{{TraceID: "t", SpanID: "s"}}})
skeleton := buildSkeleton(specs...).WithLinkedSpans(linkedSpans)
first := skeleton.LinkPage(0)
assert.Equal(t, uint64(13), first.Total)
require.Len(t, first.Items, inspectListSize)
assert.Equal(t, "consumer-00", first.Items[0].Span.SpanID)
require.NotNil(t, first.Items[0].Parent)
assert.Equal(t, "root", first.Items[0].Parent.SpanID)
assert.Equal(t, uint64(8), first.Items[0].LinkCount)
assert.Len(t, first.Items[0].Links, inspectLinkTargets)
assert.Equal(t, "consumer-09", first.Items[9].Span.SpanID)
second := skeleton.LinkPage(nextOffset(t, OverviewListLinks, first.NextCursor))
require.Len(t, second.Items, 3)
assert.Equal(t, "consumer-12", second.Items[2].Span.SpanID)
assert.Empty(t, second.NextCursor)
empty := buildSkeleton(skeletonSpec{id: "root", startMs: 0, durationMs: 1}).LinkPage(0)
assert.Equal(t, uint64(0), empty.Total)
assert.Empty(t, empty.Items)
}
func TestSlowestPage(t *testing.T) {
t.Run("SameNameUnderOneParent_CountsAsOneEntry", func(t *testing.T) {
specs := []skeletonSpec{
{id: "root", name: "POST /graphql", service: "api", startMs: 0, durationMs: 10_000},
{id: "resolve", parent: "root", name: "resolve_plans", service: "api", startMs: 1, durationMs: 9_000},
{id: "slow-single", parent: "root", name: "render", service: "api", startMs: 9_100, durationMs: 300},
}
for i := range 416 {
specs = append(specs, skeletonSpec{id: fmt.Sprintf("cost-%03d", i), parent: "resolve", name: "calculate_annual_cost", service: "pricing", startMs: int64(2 + i*20), durationMs: int64(10 + i%3), hasError: i < 3})
}
skeleton := buildSkeleton(specs...)
list := skeleton.SlowestPage(0)
require.NotEmpty(t, list.Items)
assert.Equal(t, uint64(4), list.Total)
assert.Empty(t, list.NextCursor)
group := list.Items[0]
assert.Equal(t, "calculate_annual_cost", group.Span.Name)
assert.Equal(t, uint64(416), group.SpanCount)
assert.Equal(t, uint64(3), group.ErrorSpanCount)
assert.Equal(t, uint64(12*time.Millisecond), group.Span.DurationNano)
require.NotNil(t, group.Parent)
assert.Equal(t, "resolve", group.Parent.SpanID)
assert.Equal(t, "resolve", list.Items[1].Span.SpanID)
assert.Equal(t, uint64(1), list.Items[1].SpanCount)
})
t.Run("SameNameUnderDifferentParents_StaysSeparate", func(t *testing.T) {
skeleton := buildSkeleton(
skeletonSpec{id: "root", name: "root", service: "api", startMs: 0, durationMs: 107},
skeletonSpec{id: "left", parent: "root", name: "branch", service: "api", startMs: 0, durationMs: 40},
skeletonSpec{id: "left-query", parent: "left", name: "SELECT", service: "db", startMs: 1, durationMs: 30},
skeletonSpec{id: "right", parent: "root", name: "branch", service: "worker", startMs: 50, durationMs: 40},
skeletonSpec{id: "right-query", parent: "right", name: "SELECT", service: "db", startMs: 51, durationMs: 15},
)
list := skeleton.SlowestPage(0)
assert.Equal(t, uint64(5), list.Total)
assert.Equal(t, []string{"left-query", "root", "right", "right-query", "left"}, []string{
list.Items[0].Span.SpanID, list.Items[1].Span.SpanID, list.Items[2].Span.SpanID, list.Items[3].Span.SpanID, list.Items[4].Span.SpanID,
})
})
t.Run("MoreThanOnePage_PagesOneRankingByCursor", func(t *testing.T) {
specs := []skeletonSpec{{id: "root", name: "root", service: "api", startMs: 0, durationMs: 1}}
for i := range 30 {
specs = append(specs, skeletonSpec{id: fmt.Sprintf("op-%02d", i), parent: "root", name: fmt.Sprintf("op-%02d", i), service: "api", startMs: 10, durationMs: int64(100 + i)})
}
skeleton := buildSkeleton(specs...)
seen := make([]string, 0, 31)
offset := 0
for range 5 {
list := skeleton.SlowestPage(offset)
assert.Equal(t, uint64(31), list.Total)
for _, item := range list.Items {
seen = append(seen, item.Span.SpanID)
}
if list.NextCursor == "" {
break
}
offset = nextOffset(t, OverviewListSlowest, list.NextCursor)
}
require.Len(t, seen, 31)
assert.Equal(t, "op-29", seen[0])
assert.Equal(t, "op-20", seen[9])
assert.Equal(t, "op-19", seen[10])
assert.Equal(t, "op-00", seen[29])
assert.Equal(t, "root", seen[30])
})
t.Run("EqualOwnTime_PagesInSpanOrder", func(t *testing.T) {
specs := []skeletonSpec{{id: "root", name: "root", service: "api", startMs: 0, durationMs: 1}}
for i := range 25 {
specs = append(specs, skeletonSpec{id: fmt.Sprintf("op-%02d", i), parent: "root", name: fmt.Sprintf("op-%02d", i), service: "api", startMs: int64(10 + i), durationMs: 5})
}
skeleton := buildSkeleton(specs...)
first, second, third := skeleton.SlowestPage(0), skeleton.SlowestPage(10), skeleton.SlowestPage(20)
assert.Equal(t, "op-00", first.Items[0].Span.SpanID)
assert.Equal(t, "op-09", first.Items[9].Span.SpanID)
assert.Equal(t, "op-10", second.Items[0].Span.SpanID)
assert.Equal(t, "op-19", second.Items[9].Span.SpanID)
require.Len(t, third.Items, 6)
assert.Equal(t, "op-24", third.Items[4].Span.SpanID)
assert.Equal(t, "root", third.Items[5].Span.SpanID)
assert.Empty(t, skeleton.SlowestPage(40).Items)
})
}
func TestTree(t *testing.T) {
t.Run("TraceWithinLimit_ListsEverySpanInTreeOrder", func(t *testing.T) {
skeleton := buildSkeleton(
skeletonSpec{id: "root", startMs: 0, durationMs: 100},
skeletonSpec{id: "a", parent: "root", startMs: 1, durationMs: 40},
skeletonSpec{id: "b", parent: "root", startMs: 50, durationMs: 40},
skeletonSpec{id: "a-child", parent: "a", startMs: 60, durationMs: 1},
)
tree := skeleton.tree(4)
assert.True(t, tree.Complete)
assert.Equal(t, uint64(4), tree.Total)
assert.Equal(t, []string{"root", "a", "a-child", "b"}, summaryIDs(tree.Spans))
})
t.Run("WideLevelOverLimit_ShowsItsFirstSpansInTimeOrder", func(t *testing.T) {
specs := []skeletonSpec{{id: "root", startMs: 0, durationMs: 1000}}
for i := range 200 {
spec := skeletonSpec{id: fmt.Sprintf("s-%03d", i), parent: "root", startMs: int64(i), durationMs: 1}
// A late error and a late long span do not change which spans are shown.
switch i {
case 150:
spec.hasError = true
case 170:
spec.durationMs = 900
}
specs = append(specs, spec)
}
tree := buildSkeleton(specs...).Tree()
assert.False(t, tree.Complete)
assert.Equal(t, uint64(201), tree.Total)
ids := summaryIDs(tree.Spans)
require.Len(t, ids, inspectTreeSpans)
assert.Equal(t, "root", ids[0])
assert.Equal(t, "s-000", ids[1])
assert.Equal(t, "s-048", ids[49])
assert.Equal(t, uint32(200), tree.Spans[0].ChildCount)
})
t.Run("NarrowTopLevels_DescendsUntilLimit", func(t *testing.T) {
specs := []skeletonSpec{
{id: "root", startMs: 0, durationMs: 1000},
{id: "gateway", parent: "root", startMs: 1, durationMs: 900},
{id: "handler", parent: "gateway", startMs: 2, durationMs: 800},
}
for i := range 150 {
specs = append(specs, skeletonSpec{id: fmt.Sprintf("q-%03d", i), parent: "handler", startMs: int64(3 + i), durationMs: 1})
}
tree := buildSkeleton(specs...).tree(10)
assert.Equal(t, []string{"root", "gateway", "handler", "q-000", "q-001", "q-002", "q-003", "q-004", "q-005", "q-006"}, summaryIDs(tree.Spans))
assert.Equal(t, uint32(150), tree.Spans[2].ChildCount)
})
t.Run("LevelCutAcrossParents_KeepsTreeOrder", func(t *testing.T) {
skeleton := buildSkeleton(
skeletonSpec{id: "root", startMs: 0, durationMs: 100},
skeletonSpec{id: "a", parent: "root", startMs: 1, durationMs: 40},
skeletonSpec{id: "b", parent: "root", startMs: 50, durationMs: 40},
skeletonSpec{id: "a-1", parent: "a", startMs: 2, durationMs: 1},
skeletonSpec{id: "a-2", parent: "a", startMs: 3, durationMs: 1},
skeletonSpec{id: "b-1", parent: "b", startMs: 51, durationMs: 1},
)
tree := skeleton.tree(5)
assert.False(t, tree.Complete)
assert.Equal(t, []string{"root", "a", "a-1", "a-2", "b"}, summaryIDs(tree.Spans))
})
}
func TestFitOverview(t *testing.T) {
// A root with 99 children that each fail with a 404; long names make every line large.
traceWith := func(name string) *TraceSkeleton {
specs := []skeletonSpec{{id: "root", name: "root", service: "checkout-service", startMs: 0, durationMs: 1000}}
for i := range 99 {
specs = append(specs, skeletonSpec{id: fmt.Sprintf("span-%03d", i), parent: "root", name: fmt.Sprintf("%s %03d", name, i), service: "checkout-service", startMs: int64(i), durationMs: 1, statusCode: "404"})
}
return buildSkeleton(specs...)
}
newOverview := func(skeleton *TraceSkeleton) *TraceOverview {
overview := &TraceOverview{Identity: skeleton.Identity(), UnmarkedFailures: skeleton.UnmarkedFailurePage(0), Slowest: skeleton.SlowestPage(0), Tree: skeleton.Tree()}
overview.Tree.Records = []SpanRecord{{SpanID: "root"}}
return overview
}
t.Run("OverviewOverSizeLimit_DropsRecordsThenShortensTheTree", func(t *testing.T) {
skeleton := traceWith(strings.Repeat("operation ", 40))
overview := newOverview(skeleton)
require.Greater(t, jsonChars(overview), inspectMaxResponseChars)
skeleton.FitOverview(overview)
// The tree is cut to what fits and no further: two more spans would pass the limit.
assert.LessOrEqual(t, jsonChars(overview), inspectMaxResponseChars)
assert.Greater(t, jsonChars(overview), inspectMaxResponseChars-2*jsonChars(overview.Tree.Spans[1]))
assert.Nil(t, overview.Tree.Records)
assert.Less(t, len(overview.Tree.Spans), inspectTreeSpans)
assert.False(t, overview.Tree.Complete)
assert.Equal(t, uint64(100), overview.Tree.Total)
assert.Equal(t, "root", overview.Tree.Spans[0].SpanID)
assert.Len(t, overview.Slowest.Items, inspectListSize)
})
t.Run("OverviewWithinSizeLimit_IsLeftAsItIs", func(t *testing.T) {
skeleton := traceWith("op")
overview := newOverview(skeleton)
require.LessOrEqual(t, jsonChars(overview), inspectMaxResponseChars)
skeleton.FitOverview(overview)
assert.Len(t, overview.Tree.Spans, inspectTreeSpans)
assert.NotNil(t, overview.Tree.Records)
})
}
func TestIdentity(t *testing.T) {
inputTokens, outputTokens, cost := 1200.0, 300.0, 0.02
skeleton := buildSkeleton(
skeletonSpec{id: "root", name: "POST /chat", service: "web", startMs: 0, durationMs: 100, statusCode: "200"},
skeletonSpec{id: "agent", parent: "root", name: "invoke_agent", service: "agent", startMs: 10, durationMs: 80},
skeletonSpec{id: "model", parent: "agent", name: "chat gpt", service: "agent", startMs: 20, durationMs: 50, hasError: true},
skeletonSpec{id: "tool", parent: "agent", name: "execute_tool lookup", service: "agent", startMs: 75, durationMs: 10},
).WithAISpans([]AISpanUsage{
{SpanID: "model", IsModelCall: true, InputTokens: &inputTokens, OutputTokens: &outputTokens, Cost: &cost},
{SpanID: "tool", IsToolCall: true},
{SpanID: "agent"},
})
identity := skeleton.Identity()
assert.Equal(t, uint64(100_000), identity.StartTimestampMillis)
assert.Equal(t, uint64(100*time.Millisecond), identity.DurationNano)
require.NotNil(t, identity.Root)
assert.Equal(t, "POST /chat", identity.Root.Name)
assert.Equal(t, uint64(4), identity.SpanCount)
assert.Equal(t, uint64(1), identity.ErrorSpanCount)
assert.Equal(t, uint64(2), identity.ServiceCount)
assert.Equal(t, []TraceServiceStats{
{ServiceName: "agent", SpanCount: 3, ErrorSpanCount: 1, OwnTimeNano: uint64(80 * time.Millisecond)},
{ServiceName: "web", SpanCount: 1, ErrorSpanCount: 0, OwnTimeNano: uint64(20 * time.Millisecond)},
}, identity.Services)
require.NotNil(t, identity.AI)
assert.Equal(t, uint64(1), identity.AI.ModelCallCount)
assert.Equal(t, uint64(1), identity.AI.ToolCallCount)
assert.Equal(t, uint64(1200), identity.AI.Tokens.Input)
require.NotNil(t, identity.AI.TotalCost)
assert.InDelta(t, 0.02, *identity.AI.TotalCost, 1e-9)
assert.Nil(t, buildSkeleton(skeletonSpec{id: "plain", startMs: 0, durationMs: 1}).Identity().AI)
}
func TestExpensivePage(t *testing.T) {
t.Run("WithCost_RanksByCost", func(t *testing.T) {
cheap, costly, manyTokens := 0.01, 0.5, 9000.0
skeleton := buildSkeleton(
skeletonSpec{id: "root", startMs: 0, durationMs: 100},
skeletonSpec{id: "cheap", parent: "root", startMs: 1, durationMs: 10},
skeletonSpec{id: "costly", parent: "root", startMs: 2, durationMs: 10},
).WithAISpans([]AISpanUsage{
{SpanID: "cheap", IsModelCall: true, InputTokens: &manyTokens, Cost: &cheap},
{SpanID: "costly", IsModelCall: true, Cost: &costly},
})
expensive := skeleton.ExpensivePage(0)
require.NotNil(t, expensive)
assert.Equal(t, uint64(2), expensive.Total)
assert.Equal(t, []string{"costly", "cheap"}, []string{expensive.Items[0].Span.SpanID, expensive.Items[1].Span.SpanID})
require.NotNil(t, expensive.Items[0].Parent)
assert.Equal(t, "root", expensive.Items[0].Parent.SpanID)
assert.Equal(t, uint64(9000), expensive.Items[1].InputTokens)
})
t.Run("WithoutCost_RanksByTokens", func(t *testing.T) {
few, many := 10.0, 9000.0
skeleton := buildSkeleton(
skeletonSpec{id: "small", startMs: 0, durationMs: 10},
skeletonSpec{id: "large", startMs: 1, durationMs: 10},
).WithAISpans([]AISpanUsage{
{SpanID: "small", IsModelCall: true, InputTokens: &few},
{SpanID: "large", IsModelCall: true, InputTokens: &many},
{SpanID: "not-loaded", IsModelCall: true, InputTokens: &many},
})
expensive := skeleton.ExpensivePage(0)
assert.Equal(t, uint64(2), expensive.Total)
assert.Equal(t, []string{"large", "small"}, []string{expensive.Items[0].Span.SpanID, expensive.Items[1].Span.SpanID})
assert.Nil(t, expensive.Items[0].Cost)
})
t.Run("MoreThanOnePage_PagesEqualCostsInSpanOrder", func(t *testing.T) {
cost := 0.25
specs := []skeletonSpec{{id: "root", startMs: 0, durationMs: 1000}}
usage := make([]AISpanUsage, 0, 14)
for i := range 14 {
id := fmt.Sprintf("call-%02d", i)
specs = append(specs, skeletonSpec{id: id, parent: "root", startMs: int64(i + 1), durationMs: 1})
// Given in reverse so the page order has to come from the spans, not from the input.
usage = append([]AISpanUsage{{SpanID: id, IsModelCall: true, Cost: &cost}}, usage...)
}
skeleton := buildSkeleton(specs...).WithAISpans(usage)
first := skeleton.ExpensivePage(0)
second := skeleton.ExpensivePage(nextOffset(t, OverviewListExpensive, first.NextCursor))
assert.Equal(t, uint64(14), first.Total)
assert.Equal(t, "call-00", first.Items[0].Span.SpanID)
assert.Equal(t, "call-09", first.Items[9].Span.SpanID)
require.Len(t, second.Items, 4)
assert.Equal(t, "call-13", second.Items[3].Span.SpanID)
assert.Empty(t, second.NextCursor)
})
t.Run("NoGenAISpans_ReturnsNothing", func(t *testing.T) {
assert.Nil(t, buildSkeleton(skeletonSpec{id: "plain", startMs: 0, durationMs: 1}).ExpensivePage(0))
})
}

View File

@@ -0,0 +1,454 @@
package spantypes
import (
"encoding/json"
"slices"
"sort"
"time"
)
const (
skeletonFlagError uint8 = 1 << iota
skeletonFlagExceptionEvent
skeletonFlagMissingParent
)
const noParent int32 = -1
// SkeletonSpan is the ClickHouse scan struct for the columns a TraceSkeleton is built from.
type SkeletonSpan struct {
SpanID string `ch:"span_id"`
ParentSpanID string `ch:"parent_span_id"`
StartTime time.Time `ch:"timestamp"`
DurationNano uint64 `ch:"duration_nano"`
HasError bool `ch:"has_error"`
Name string `ch:"name"`
ServiceName string `ch:"resource_string_service$$name"`
KindString string `ch:"kind_string"`
ResponseStatusCode string `ch:"response_status_code"`
HasExceptionEvent bool `ch:"has_exception_event"`
}
// AISpanUsage is the token and cost usage of one gen_ai span.
type AISpanUsage struct {
SpanID string `ch:"span_id"`
IsModelCall bool `ch:"is_model_call"`
IsToolCall bool `ch:"is_tool_call"`
InputTokens *float64 `ch:"input_tokens_value"`
OutputTokens *float64 `ch:"output_tokens_value"`
CacheReadTokens *float64 `ch:"cache_read_tokens_value"`
CacheWriteTokens *float64 `ch:"cache_write_tokens_value"`
ReasoningTokens *float64 `ch:"reasoning_tokens_value"`
Cost *float64 `ch:"total_cost_value"`
}
// LinkedSpan is a span that links to other spans, in its own trace or another one.
type LinkedSpan struct {
SpanID string
Links []Link
}
// NewLinkedSpan reads the links out of a span's stored references; ok is false when it has none.
func NewLinkedSpan(spanID, references string) (LinkedSpan, bool) {
refs := []OtelSpanRef{}
if err := json.Unmarshal([]byte(references), &refs); err != nil {
return LinkedSpan{}, false
}
linked := LinkedSpan{SpanID: spanID, Links: make([]Link, 0, len(refs))}
for _, ref := range refs {
// The parent is stored as a CHILD_OF reference beside the links.
if ref.RefType != "CHILD_OF" {
linked.Links = append(linked.Links, Link{TraceID: ref.TraceId, SpanID: ref.SpanId})
}
}
return linked, len(linked.Links) > 0
}
// TraceSkeleton holds the tree of a trace as parallel arrays indexed by span position, in start
// time order. It is read-only once built.
type TraceSkeleton struct {
TraceID string
// SpanIDs holds every span id back to back; span i is SpanIDs[IDOffsets[i]:IDOffsets[i+1]].
SpanIDs string
IDOffsets []uint32
// Parents holds the position of each span's parent, or -1 for a root.
Parents []int32
StartNano []int64
DurationNano []uint64
Flags []uint8
// Names, Services, Kinds and StatusCodes index into Strings.
Names []uint32
Services []uint32
Kinds []uint32
StatusCodes []uint32
Strings []string
AISpans []AISpanUsage
// LinkedSpans holds the spans with links, in span order.
LinkedSpans []LinkedSpan
// Children of span i are childList[childStart[i]:childStart[i+1]], in span order.
childStart []int32
childList []int32
roots []int32
// preorder lists span positions depth first, roots and siblings in span order.
preorder []int32
level []uint32
// subtree counts the spans below each span, the span itself excluded.
subtree []uint32
ownTime []uint64
// idOrder lists span positions sorted by span id, for lookups by id.
idOrder []int32
startNs int64
endNs int64
errCount uint64
}
// TraceSkeletonBuilder collects spans in (start time, name, span id) order.
type TraceSkeletonBuilder struct {
skeleton *TraceSkeleton
ids []byte
parentIDs []string
interned map[string]uint32
}
func NewTraceSkeletonBuilder(traceID string, sizeHint int) *TraceSkeletonBuilder {
return &TraceSkeletonBuilder{
skeleton: &TraceSkeleton{
TraceID: traceID,
IDOffsets: make([]uint32, 1, sizeHint+1),
Parents: make([]int32, 0, sizeHint),
StartNano: make([]int64, 0, sizeHint),
DurationNano: make([]uint64, 0, sizeHint),
Flags: make([]uint8, 0, sizeHint),
Names: make([]uint32, 0, sizeHint),
Services: make([]uint32, 0, sizeHint),
Kinds: make([]uint32, 0, sizeHint),
StatusCodes: make([]uint32, 0, sizeHint),
AISpans: []AISpanUsage{},
LinkedSpans: []LinkedSpan{},
},
ids: make([]byte, 0, sizeHint*16),
parentIDs: make([]string, 0, sizeHint),
interned: make(map[string]uint32),
}
}
func (b *TraceSkeletonBuilder) Add(span *SkeletonSpan) {
s := b.skeleton
b.ids = append(b.ids, span.SpanID...)
s.IDOffsets = append(s.IDOffsets, uint32(len(b.ids)))
b.parentIDs = append(b.parentIDs, span.ParentSpanID)
s.StartNano = append(s.StartNano, span.StartTime.UnixNano())
s.DurationNano = append(s.DurationNano, span.DurationNano)
var flags uint8
if span.HasError {
flags |= skeletonFlagError
}
if span.HasExceptionEvent {
flags |= skeletonFlagExceptionEvent
}
s.Flags = append(s.Flags, flags)
s.Names = append(s.Names, b.intern(span.Name))
s.Services = append(s.Services, b.intern(span.ServiceName))
s.Kinds = append(s.Kinds, b.intern(span.KindString))
s.StatusCodes = append(s.StatusCodes, b.intern(span.ResponseStatusCode))
}
func (b *TraceSkeletonBuilder) Len() int {
return len(b.parentIDs)
}
// Build links each span to its parent and computes the tree fields. The builder cannot be reused.
func (b *TraceSkeletonBuilder) Build() *TraceSkeleton {
s := b.skeleton
s.SpanIDs = string(b.ids)
s.buildIDOrder()
n := len(b.parentIDs)
s.Parents = s.Parents[:0]
for i := range n {
parent := noParent
if parentID := b.parentIDs[i]; parentID != "" {
if idx, ok := s.Index(parentID); ok && idx != int32(i) {
parent = idx
} else {
s.Flags[i] |= skeletonFlagMissingParent
}
}
s.Parents = append(s.Parents, parent)
}
b.parentIDs, b.interned, b.ids = nil, nil, nil
s.derive()
return s
}
func (b *TraceSkeletonBuilder) intern(value string) uint32 {
if idx, ok := b.interned[value]; ok {
return idx
}
idx := uint32(len(b.skeleton.Strings))
b.skeleton.Strings = append(b.skeleton.Strings, value)
b.interned[value] = idx
return idx
}
func (s *TraceSkeleton) Len() int {
return len(s.Parents)
}
func (s *TraceSkeleton) SpanID(i int32) string {
return s.SpanIDs[s.IDOffsets[i]:s.IDOffsets[i+1]]
}
// Index returns the position of the span with the given id.
func (s *TraceSkeleton) Index(spanID string) (int32, bool) {
pos, found := sort.Find(len(s.idOrder), func(k int) int {
id := s.SpanID(s.idOrder[k])
switch {
case spanID < id:
return -1
case spanID > id:
return 1
}
return 0
})
if !found {
return 0, false
}
return s.idOrder[pos], true
}
func (s *TraceSkeleton) Children(i int32) []int32 {
return s.childList[s.childStart[i]:s.childStart[i+1]]
}
func (s *TraceSkeleton) HasError(i int32) bool {
return s.Flags[i]&skeletonFlagError != 0
}
func (s *TraceSkeleton) HasExceptionEvent(i int32) bool {
return s.Flags[i]&skeletonFlagExceptionEvent != 0
}
func (s *TraceSkeleton) HasMissingParent(i int32) bool {
return s.Flags[i]&skeletonFlagMissingParent != 0
}
func (s *TraceSkeleton) ResponseStatusCode(i int32) string {
return s.Strings[s.StatusCodes[i]]
}
func (s *TraceSkeleton) HasMissingSpans() bool {
for _, root := range s.roots {
if s.HasMissingParent(root) {
return true
}
}
return false
}
// PrimaryRoot is the earliest span without a parent, or the earliest orphan when every root's
// parent is missing.
func (s *TraceSkeleton) PrimaryRoot() (int32, bool) {
if len(s.roots) == 0 {
return 0, false
}
for _, root := range s.roots {
if !s.HasMissingParent(root) {
return root, true
}
}
return s.roots[0], true
}
// Summary returns the short form of span i.
func (s *TraceSkeleton) Summary(i int32) SpanSummary {
summary := SpanSummary{
SpanID: s.SpanID(i),
Name: s.Strings[s.Names[i]],
ServiceName: s.Strings[s.Services[i]],
Kind: s.Strings[s.Kinds[i]],
StartOffsetNano: uint64(s.StartNano[i] - s.startNs),
DurationNano: s.DurationNano[i],
OwnTimeNano: s.ownTime[i],
ResponseStatusCode: s.ResponseStatusCode(i),
HasError: s.HasError(i),
Level: s.level[i],
ChildCount: uint32(s.childStart[i+1] - s.childStart[i]),
SubtreeSpanCount: s.subtree[i],
}
if parent := s.Parents[i]; parent != noParent {
summary.ParentSpanID = s.SpanID(parent)
}
return summary
}
// WithAISpans returns a copy of the skeleton carrying the usage of its gen_ai spans.
func (s *TraceSkeleton) WithAISpans(aiSpans []AISpanUsage) *TraceSkeleton {
cp := *s
cp.AISpans = aiSpans
if cp.AISpans == nil {
cp.AISpans = []AISpanUsage{}
}
return &cp
}
// WithLinkedSpans returns a copy of the skeleton carrying its spans that have links, in span order.
func (s *TraceSkeleton) WithLinkedSpans(linkedSpans []LinkedSpan) *TraceSkeleton {
type positioned struct {
index int32
linked LinkedSpan
}
known := make([]positioned, 0, len(linkedSpans))
for _, linked := range linkedSpans {
if idx, ok := s.Index(linked.SpanID); ok {
known = append(known, positioned{index: idx, linked: linked})
}
}
slices.SortFunc(known, func(a, b positioned) int { return int(a.index) - int(b.index) })
cp := *s
cp.LinkedSpans = make([]LinkedSpan, 0, len(known))
for _, item := range known {
cp.LinkedSpans = append(cp.LinkedSpans, item.linked)
}
return &cp
}
func (s *TraceSkeleton) buildIDOrder() {
n := len(s.IDOffsets) - 1
s.idOrder = make([]int32, n)
for i := range s.idOrder {
s.idOrder[i] = int32(i)
}
slices.SortFunc(s.idOrder, func(a, b int32) int {
idA, idB := s.SpanID(a), s.SpanID(b)
switch {
case idA < idB:
return -1
case idA > idB:
return 1
}
return 0
})
}
func (s *TraceSkeleton) derive() {
n := s.Len()
s.startNs, s.endNs, s.errCount = 0, 0, 0
for i := range n {
if i == 0 || s.StartNano[i] < s.startNs {
s.startNs = s.StartNano[i]
}
s.endNs = max(s.endNs, s.StartNano[i]+int64(s.DurationNano[i]))
if s.Flags[i]&skeletonFlagError != 0 {
s.errCount++
}
}
s.buildChildren()
// A span left unvisited sits on a parent cycle; cutting its parent link makes it a root.
if s.walk() < n {
visited := make([]bool, n)
for _, i := range s.preorder {
visited[i] = true
}
for i := range n {
if visited[i] {
continue
}
s.Parents[i] = noParent
s.Flags[i] |= skeletonFlagMissingParent
stack := []int32{int32(i)}
for len(stack) > 0 {
top := stack[len(stack)-1]
stack = stack[:len(stack)-1]
if visited[top] {
continue
}
visited[top] = true
stack = append(stack, s.Children(top)...)
}
}
s.buildChildren()
s.walk()
}
s.subtree = make([]uint32, n)
for k := n - 1; k >= 0; k-- {
i := s.preorder[k]
if parent := s.Parents[i]; parent != noParent {
s.subtree[parent] += s.subtree[i] + 1
}
}
s.ownTime = make([]uint64, n)
for i := range n {
s.ownTime[i] = s.DurationNano[i] - s.childrenCoverNano(int32(i))
}
}
func (s *TraceSkeleton) buildChildren() {
n := s.Len()
s.childStart = make([]int32, n+1)
s.roots = s.roots[:0]
for i := range n {
if parent := s.Parents[i]; parent != noParent {
s.childStart[parent+1]++
} else {
s.roots = append(s.roots, int32(i))
}
}
for i := range n {
s.childStart[i+1] += s.childStart[i]
}
s.childList = make([]int32, s.childStart[n])
next := slices.Clone(s.childStart[:n])
for i := range n {
if parent := s.Parents[i]; parent != noParent {
s.childList[next[parent]] = int32(i)
next[parent]++
}
}
}
// walk fills preorder and level from the roots and returns the number of spans reached.
func (s *TraceSkeleton) walk() int {
n := s.Len()
s.preorder = make([]int32, 0, n)
s.level = make([]uint32, n)
stack := make([]int32, 0, 64)
for r := len(s.roots) - 1; r >= 0; r-- {
stack = append(stack, s.roots[r])
}
for len(stack) > 0 {
i := stack[len(stack)-1]
stack = stack[:len(stack)-1]
s.preorder = append(s.preorder, i)
children := s.Children(i)
for c := len(children) - 1; c >= 0; c-- {
s.level[children[c]] = s.level[i] + 1
stack = append(stack, children[c])
}
}
return len(s.preorder)
}
// childrenCoverNano is the time of span i its children cover, overlapping children counted once.
func (s *TraceSkeleton) childrenCoverNano(i int32) uint64 {
start := s.StartNano[i]
end := start + int64(s.DurationNano[i])
covered, cursor := int64(0), start
for _, child := range s.Children(i) {
childStart := max(s.StartNano[child], cursor)
childEnd := min(s.StartNano[child]+int64(s.DurationNano[child]), end)
if childEnd <= childStart {
continue
}
covered += childEnd - childStart
cursor = childEnd
}
return uint64(covered)
}

View File

@@ -0,0 +1,217 @@
package spantypes
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type skeletonSpec struct {
id, parent, name, service string
startMs, durationMs int64
hasError, hasException bool
statusCode string
}
// buildSkeleton adds the spans in the given order, which stands for the store's start time order.
func buildSkeleton(specs ...skeletonSpec) *TraceSkeleton {
builder := NewTraceSkeletonBuilder("trace", len(specs))
for _, spec := range specs {
builder.Add(&SkeletonSpan{
SpanID: spec.id,
ParentSpanID: spec.parent,
StartTime: time.Unix(100, 0).Add(time.Duration(spec.startMs) * time.Millisecond),
DurationNano: uint64(spec.durationMs) * uint64(time.Millisecond),
HasError: spec.hasError,
Name: spec.name,
ServiceName: spec.service,
KindString: "Internal",
ResponseStatusCode: spec.statusCode,
HasExceptionEvent: spec.hasException,
})
}
return builder.Build()
}
func ownTimeMs(t *testing.T, skeleton *TraceSkeleton, spanID string) int64 {
t.Helper()
idx, ok := skeleton.Index(spanID)
require.True(t, ok, "span %s", spanID)
return int64(skeleton.ownTime[idx] / uint64(time.Millisecond))
}
func TestTraceSkeletonOwnTime(t *testing.T) {
testCases := []struct {
name string
specs []skeletonSpec
expected int64
}{
{
name: "SequentialChildren_SubtractsEach",
specs: []skeletonSpec{
{id: "root", startMs: 0, durationMs: 100},
{id: "a", parent: "root", startMs: 10, durationMs: 20},
{id: "b", parent: "root", startMs: 50, durationMs: 30},
},
expected: 50,
},
{
name: "ParallelChildren_CountsOverlapOnce",
specs: []skeletonSpec{
{id: "root", startMs: 0, durationMs: 100},
{id: "a", parent: "root", startMs: 10, durationMs: 40},
{id: "b", parent: "root", startMs: 20, durationMs: 40},
{id: "c", parent: "root", startMs: 30, durationMs: 10},
},
expected: 50,
},
{
name: "ChildOutlivesParent_ClipsToParentEnd",
specs: []skeletonSpec{
{id: "root", startMs: 0, durationMs: 100},
{id: "a", parent: "root", startMs: 80, durationMs: 500},
},
expected: 80,
},
{
name: "ChildStartsBeforeParent_ClipsToParentStart",
specs: []skeletonSpec{
{id: "early", parent: "root", startMs: 0, durationMs: 30},
{id: "root", startMs: 10, durationMs: 100},
},
expected: 80,
},
{
name: "ChildCoversParent_LeavesZero",
specs: []skeletonSpec{
{id: "early", parent: "root", startMs: 0, durationMs: 500},
{id: "root", startMs: 10, durationMs: 100},
},
expected: 0,
},
{
name: "NoChildren_EqualsDuration",
specs: []skeletonSpec{
{id: "root", startMs: 0, durationMs: 70},
},
expected: 70,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
skeleton := buildSkeleton(testCase.specs...)
assert.Equal(t, testCase.expected, ownTimeMs(t, skeleton, "root"))
})
}
}
func TestTraceSkeletonTree(t *testing.T) {
skeleton := buildSkeleton(
skeletonSpec{id: "root", name: "GET /", service: "web", startMs: 0, durationMs: 100},
skeletonSpec{id: "a", parent: "root", name: "a", service: "web", startMs: 10, durationMs: 30},
skeletonSpec{id: "a1", parent: "a", name: "a1", service: "db", startMs: 15, durationMs: 5, hasError: true},
skeletonSpec{id: "b", parent: "root", name: "b", service: "web", startMs: 50, durationMs: 10},
skeletonSpec{id: "orphan", parent: "gone", name: "orphan", service: "worker", startMs: 60, durationMs: 10},
)
order := make([]string, 0, skeleton.Len())
for _, i := range skeleton.preorder {
order = append(order, skeleton.SpanID(i))
}
assert.Equal(t, []string{"root", "a", "a1", "b", "orphan"}, order)
assert.Len(t, skeleton.roots, 2)
assert.True(t, skeleton.HasMissingSpans())
assert.Equal(t, uint64(1), skeleton.errCount)
root, ok := skeleton.PrimaryRoot()
require.True(t, ok)
rootSummary := skeleton.Summary(root)
assert.Equal(t, "root", rootSummary.SpanID)
assert.Equal(t, uint32(2), rootSummary.ChildCount)
assert.Equal(t, uint32(3), rootSummary.SubtreeSpanCount)
leaf, ok := skeleton.Index("a1")
require.True(t, ok)
leafSummary := skeleton.Summary(leaf)
assert.Equal(t, uint32(2), leafSummary.Level)
assert.Equal(t, "a", leafSummary.ParentSpanID)
assert.Equal(t, uint64(15*time.Millisecond), leafSummary.StartOffsetNano)
assert.True(t, leafSummary.HasError)
assert.Equal(t, "db", leafSummary.ServiceName)
_, ok = skeleton.Index("gone")
assert.False(t, ok)
}
func TestTraceSkeletonPrimaryRoot(t *testing.T) {
testCases := []struct {
name string
specs []skeletonSpec
expected string
}{
{
name: "OrphanStartsFirst_PrefersSpanWithoutParent",
specs: []skeletonSpec{
{id: "orphan", parent: "gone", startMs: 0, durationMs: 10},
{id: "root", startMs: 5, durationMs: 100},
},
expected: "root",
},
{
name: "OnlyOrphans_TakesEarliest",
specs: []skeletonSpec{
{id: "first", parent: "gone", startMs: 0, durationMs: 10},
{id: "second", parent: "missing", startMs: 5, durationMs: 100},
},
expected: "first",
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
skeleton := buildSkeleton(testCase.specs...)
root, ok := skeleton.PrimaryRoot()
require.True(t, ok)
assert.Equal(t, testCase.expected, skeleton.SpanID(root))
})
}
}
func TestTraceSkeletonParentCycle(t *testing.T) {
testCases := []struct {
name string
specs []skeletonSpec
}{
{
name: "TwoSpansPointAtEachOther",
specs: []skeletonSpec{
{id: "root", startMs: 0, durationMs: 10},
{id: "x", parent: "y", startMs: 1, durationMs: 5},
{id: "y", parent: "x", startMs: 2, durationMs: 5},
{id: "under-y", parent: "y", startMs: 3, durationMs: 1},
},
},
{
name: "SpanIsItsOwnParent",
specs: []skeletonSpec{
{id: "self", parent: "self", startMs: 0, durationMs: 10},
{id: "child", parent: "self", startMs: 1, durationMs: 5},
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
skeleton := buildSkeleton(testCase.specs...)
assert.Len(t, skeleton.preorder, len(testCase.specs))
total := 0
for _, root := range skeleton.roots {
total += int(skeleton.Summary(root).SubtreeSpanCount) + 1
}
assert.Equal(t, len(testCase.specs), total)
})
}
}

View File

@@ -0,0 +1,153 @@
package spantypes
import (
"fmt"
"slices"
"strings"
"unicode/utf8"
)
const (
spanRecordValueChars = 500
spanRecordChars = 20_000
spanRecordShrunkChars = 100
spanRecordMinimumChars = 20
truncationMark = "…"
)
// TruncatedField names a value of a span record that was cut, with its full length in characters.
type TruncatedField struct {
Field string `json:"field" required:"true"`
Length uint64 `json:"length" required:"true"`
}
// SpanRecord is everything stored on a span. Long values are cut and listed in Truncated.
type SpanRecord struct {
SpanID string `json:"spanId" required:"true"`
StatusCode string `json:"statusCode" required:"true"`
StatusMessage string `json:"statusMessage" required:"true"`
Attributes map[string]any `json:"attributes" required:"true" nullable:"false"`
Resource map[string]string `json:"resource" required:"true" nullable:"false"`
Events []EventV2 `json:"events" required:"true" nullable:"false"`
// Links holds the spans this span links to, in this or another trace.
Links []Link `json:"links" required:"true" nullable:"false"`
Truncated []TruncatedField `json:"truncated" required:"true" nullable:"false"`
}
// NewSpanRecord cuts values over 500 characters, and shorter still while the record is over its size limit.
func NewSpanRecord(item *StorableSpan) SpanRecord {
record := SpanRecord{
SpanID: item.SpanID,
StatusCode: item.StatusCodeString,
StatusMessage: item.StatusMessage,
Attributes: item.Attributes(),
Resource: make(map[string]string, len(item.ResourcesString)),
Events: make([]EventV2, 0, len(item.Events)),
Links: make([]Link, 0),
Truncated: []TruncatedField{},
}
for key, value := range item.ResourcesString {
record.Resource[key] = value
}
for _, event := range item.UnmarshalledEvents() {
record.Events = append(record.Events, EventV2{Name: event.Name, TimeUnixNano: event.TimeUnixNano, Attributes: event.AttributeMap, IsError: event.IsError})
}
for _, ref := range item.UnmarshalledRefs() {
if ref.RefType != "CHILD_OF" {
record.Links = append(record.Links, Link{TraceID: ref.TraceId, SpanID: ref.SpanId})
}
}
record.cut(spanRecordValueChars)
for _, limit := range []int{spanRecordShrunkChars, spanRecordMinimumChars} {
if jsonChars(record) <= spanRecordChars {
break
}
record.cut(limit)
}
slices.SortFunc(record.Truncated, func(a, b TruncatedField) int { return strings.Compare(a.Field, b.Field) })
return record
}
// cut shortens every string value longer than limit and records its full length once.
func (r *SpanRecord) cut(limit int) {
noted := make(map[string]struct{}, len(r.Truncated))
for _, field := range r.Truncated {
noted[field.Field] = struct{}{}
}
note := func(field string, length int) {
if _, ok := noted[field]; !ok {
noted[field] = struct{}{}
r.Truncated = append(r.Truncated, TruncatedField{Field: field, Length: uint64(length)})
}
}
if cutValue, length, ok := cutString(r.StatusMessage, limit); ok {
r.StatusMessage = cutValue
note("statusMessage", length)
}
for key, value := range r.Attributes {
if str, isString := value.(string); isString {
if cutValue, length, ok := cutString(str, limit); ok {
r.Attributes[key] = cutValue
note("attributes."+key, length)
}
}
}
for key, value := range r.Resource {
if cutValue, length, ok := cutString(value, limit); ok {
r.Resource[key] = cutValue
note("resource."+key, length)
}
}
for idx := range r.Events {
for key, value := range r.Events[idx].Attributes {
if str, isString := value.(string); isString {
if cutValue, length, ok := cutString(str, limit); ok {
r.Events[idx].Attributes[key] = cutValue
note(fmt.Sprintf("events[%d].attributes.%s", idx, key), length)
}
}
}
}
}
// cutString returns value shortened to limit characters, and its length when it was longer or already cut.
func cutString(value string, limit int) (string, int, bool) {
if len(value) <= limit {
return value, 0, false
}
length := utf8.RuneCountInString(value)
if length <= limit {
return value, 0, false
}
cut := 0
for idx := range value {
if cut == limit {
return value[:idx] + truncationMark, length, true
}
cut++
}
return value, 0, false
}
// SmallTraceRecords returns the full record of every span when the trace is small enough for all
// of them to fit in the response; ok is false otherwise.
func SmallTraceRecords(spans []StorableSpan) ([]SpanRecord, bool) {
records := make([]SpanRecord, 0, len(spans))
chars := 0
for i := range spans {
record := NewSpanRecord(&spans[i])
chars += jsonChars(record)
if chars > inspectSmallTraceChars {
return nil, false
}
records = append(records, record)
}
return records, true
}
// IsSmallTrace reports whether a trace has few enough spans to try returning their records.
func IsSmallTrace(spanCount int) bool {
return spanCount <= inspectTreeSpans
}

View File

@@ -0,0 +1,84 @@
package spantypes
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func recordSpan(spanID string, attributes map[string]string) StorableSpan {
return StorableSpan{
SpanID: spanID,
Name: "POST /charge",
ServiceName: "payments",
SpanKind: "Server",
StatusCodeString: "Error",
StatusMessage: "worldpay declined the card",
AttributesString: attributes,
AttributesNumber: map[string]float64{"http.response.status_code": 502},
ResourcesString: map[string]string{"service.name": "payments"},
Events: []string{`{"name":"exception","timeUnixNano":5,"attributeMap":{"exception.message":"card declined"},"isError":true}`},
References: `[{"traceId":"trace","spanId":"parent","refType":"CHILD_OF"},{"traceId":"other-trace","spanId":"producer","refType":"FOLLOWS_FROM"}]`,
}
}
func TestNewSpanRecord(t *testing.T) {
longStatement := strings.Repeat("SELECT 1; ", 120)
t.Run("LongValue_IsCutAndListed", func(t *testing.T) {
span := recordSpan("span-a", map[string]string{"db.statement": longStatement, "http.route": "/charge"})
record := NewSpanRecord(&span)
assert.Equal(t, "Error", record.StatusCode)
assert.Equal(t, "/charge", record.Attributes["http.route"])
assert.InDelta(t, 502, record.Attributes["http.response.status_code"], 0)
assert.Len(t, []rune(record.Attributes["db.statement"].(string)), spanRecordValueChars+1)
assert.Equal(t, []TruncatedField{{Field: "attributes.db.statement", Length: uint64(len(longStatement))}}, record.Truncated)
require.Len(t, record.Events, 1)
assert.True(t, record.Events[0].IsError)
assert.Equal(t, []Link{{TraceID: "other-trace", SpanID: "producer"}}, record.Links)
})
t.Run("MultiByteValue_IsCutOnACharacterBoundary", func(t *testing.T) {
span := recordSpan("span-c", map[string]string{"note": strings.Repeat("é", 600)})
record := NewSpanRecord(&span)
assert.Equal(t, strings.Repeat("é", spanRecordValueChars)+truncationMark, record.Attributes["note"])
assert.Equal(t, uint64(600), record.Truncated[0].Length)
})
t.Run("RecordOverBudget_ShrinksValuesFurther", func(t *testing.T) {
attributes := make(map[string]string, 80)
for i := range 80 {
attributes["key."+strings.Repeat("k", i+1)] = strings.Repeat("v", 480)
}
span := recordSpan("span-d", attributes)
record := NewSpanRecord(&span)
assert.LessOrEqual(t, jsonChars(record), spanRecordChars)
assert.Len(t, record.Truncated, 80)
assert.Equal(t, uint64(480), record.Truncated[0].Length)
})
}
func TestSmallTraceRecords(t *testing.T) {
small := []StorableSpan{recordSpan("span-a", map[string]string{"k": "v"}), recordSpan("span-b", map[string]string{"k": "v"})}
records, ok := SmallTraceRecords(small)
require.True(t, ok)
assert.Len(t, records, 2)
large := make([]StorableSpan, 0, 60)
for range 60 {
large = append(large, recordSpan("span", map[string]string{"payload": strings.Repeat("x", 450)}))
}
_, ok = SmallTraceRecords(large)
assert.False(t, ok)
assert.True(t, IsSmallTrace(inspectTreeSpans))
assert.False(t, IsSmallTrace(inspectTreeSpans+1))
}

View File

@@ -0,0 +1,288 @@
import json
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import pytest
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.querier import get_attribute_backend_now
from fixtures.traces import TraceIdGenerator, Traces, TracesEvent, TracesKind, TracesLink, TracesRefType, TracesStatusCode
OVERVIEW_LISTS = ("errors", "unmarkedFailures", "slowest", "expensive", "links")
LIST_SIZE = 10
def get_inspect(signoz: types.SigNoz, token: str, trace_id: str, params: dict | None = None) -> requests.Response:
return requests.get(
signoz.self.host_configs["8080"].get(f"/api/v1/traces/{trace_id}/inspect"),
params=params,
headers={"authorization": f"Bearer {token}"},
timeout=10,
)
def get_overview(signoz: types.SigNoz, token: str, trace_id: str, params: dict | None = None) -> dict:
response = get_inspect(signoz, token, trace_id, params)
assert response.status_code == HTTPStatus.OK, f"{params}: {response.text}"
data = response.json()["data"]
assert data["type"] == "overview"
return data["overview"]
@pytest.mark.parametrize("attribute_backend", ["map", "json"])
def test_inspect_overview(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
attribute_backend: str,
) -> None:
"""The overview of a small trace: identity with per-service totals and gen_ai usage, where the
error started, the failures not marked as errors, span links, the most expensive gen_ai spans and
the whole tree with every span's record. Attributes are written to one layout only, so a read
from the wrong column loses the tokens, the cost and the record's attributes."""
now = get_attribute_backend_now(signoz, attribute_backend).replace(second=0, microsecond=0)
write_mode = "json_only" if attribute_backend == "json" else "legacy_only"
trace_id = TraceIdGenerator.trace_id()
other_trace_id = TraceIdGenerator.trace_id()
names = ("root", "place_order", "charge_card", "inventory_http", "inventory_grpc", "render", "consume", "model", "tool")
ids = {name: TraceIdGenerator.span_id() for name in names}
gateway, orders, agent = (f"td-inspect-{service}-{attribute_backend}" for service in ("gateway", "orders", "agent"))
def span(key: str, parent: str, name: str, service: str, offset_ms: int, duration_ms: int, **kwargs) -> Traces:
return Traces(
timestamp=now - timedelta(seconds=10) + timedelta(milliseconds=offset_ms),
duration=timedelta(milliseconds=duration_ms),
trace_id=trace_id,
span_id=ids[key],
parent_span_id=ids[parent] if parent else "",
name=name,
resources={"service.name": service},
attribute_write_mode=write_mode,
**kwargs,
)
render = span("render", "root", "render_receipt", gateway, 700, 50, events=[TracesEvent("exception", now, {"exception.message": "template missing, fell back"})])
# The collector marks an exception event with isError; the fixture does not write it.
render._events_db = [json.dumps({"name": "exception", "timeUnixNano": 1, "attributeMap": {"exception.message": "template missing, fell back"}, "isError": True}, separators=(",", ":"))] # pylint: disable=protected-access
insert_traces(
[
span("root", "", "POST /checkout", gateway, 0, 1000, kind=TracesKind.SPAN_KIND_SERVER, status_code=TracesStatusCode.STATUS_CODE_ERROR, attributes={"http.response.status_code": 500}),
# the error is carried up from charge_card through place_order to the root
span("place_order", "root", "place_order", orders, 10, 600, status_code=TracesStatusCode.STATUS_CODE_ERROR),
span("charge_card", "place_order", "charge_card", orders, 20, 300, kind=TracesKind.SPAN_KIND_CLIENT, status_code=TracesStatusCode.STATUS_CODE_ERROR, status_message="card declined", attributes={"payment.provider": "worldpay"}),
# three failures that do not carry the error flag
span("inventory_http", "place_order", "GET /inventory", orders, 330, 40, kind=TracesKind.SPAN_KIND_CLIENT, attributes={"http.response.status_code": 404}),
span("inventory_grpc", "place_order", "inventory.Check", orders, 380, 40, kind=TracesKind.SPAN_KIND_CLIENT, attributes={"rpc.grpc.status_code": 5}),
render,
# a link out to another trace
span("consume", "root", "consume order.created", orders, 760, 20, kind=TracesKind.SPAN_KIND_CONSUMER, links=[TracesLink(trace_id=other_trace_id, span_id="aaaaaaaaaaaaaaaa", ref_type=TracesRefType.REF_TYPE_FOLLOWS_FROM)]),
span("model", "root", "chat gpt-4o-mini", agent, 800, 100, kind=TracesKind.SPAN_KIND_CLIENT, attributes={"gen_ai.request.model": "gpt-4o-mini", "gen_ai.usage.input_tokens": 100, "gen_ai.usage.output_tokens": 20, "signoz.gen_ai.usage.tokens.cost": 0.01}),
span("tool", "root", "execute_tool", agent, 910, 30, attributes={"gen_ai.tool.name": "get_weather"}),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
overview = get_overview(signoz, token, trace_id)
identity = overview["identity"]
assert identity["traceId"] == trace_id
assert identity["spanCount"] == 9
assert identity["errorSpanCount"] == 3
assert identity["rootCount"] == 1
assert identity["hasMissingSpans"] is False
assert identity["durationNano"] == 1_000_000_000
assert identity["root"]["spanId"] == ids["root"]
assert identity["root"]["name"] == "POST /checkout"
assert identity["root"]["responseStatusCode"] == "500"
assert identity["serviceCount"] == 3
assert {service["serviceName"]: (service["spanCount"], service["errorSpanCount"]) for service in identity["services"]} == {gateway: (2, 1), orders: (5, 2), agent: (2, 0)}
assert identity["ai"]["modelCallCount"] == 1
assert identity["ai"]["toolCallCount"] == 1
assert identity["ai"]["tokens"] == {"input": 100, "output": 20, "cacheRead": 0, "cacheWrite": 0, "reasoning": 0}
assert identity["ai"]["totalCost"] == pytest.approx(0.01)
# one entry for the error, at the span that threw it
errors = overview["errors"]
assert errors["total"] == 1
assert "nextCursor" not in errors
assert errors["items"][0]["span"]["spanId"] == ids["charge_card"]
assert errors["items"][0]["parent"]["spanId"] == ids["place_order"]
assert errors["items"][0]["statusMessage"] == "card declined"
assert errors["items"][0]["errorAncestorCount"] == 2
unmarked = overview["unmarkedFailures"]
assert unmarked["total"] == 3
assert {item["span"]["spanId"]: item["reason"] for item in unmarked["items"]} == {ids["inventory_http"]: "http_4xx", ids["inventory_grpc"]: "grpc_status", ids["render"]: "exception_event"}
links = overview["links"]
assert links["total"] == 1
assert links["items"][0]["span"]["spanId"] == ids["consume"]
assert links["items"][0]["parent"]["spanId"] == ids["root"]
assert links["items"][0]["linkCount"] == 1
assert links["items"][0]["links"] == [{"traceId": other_trace_id, "spanId": "aaaaaaaaaaaaaaaa"}]
expensive = overview["expensive"]
assert expensive["total"] == 2
assert expensive["items"][0]["span"]["spanId"] == ids["model"]
assert expensive["items"][0]["cost"] == pytest.approx(0.01)
assert (expensive["items"][0]["inputTokens"], expensive["items"][0]["outputTokens"]) == (100, 20)
# own time is the duration minus what the children cover: 300 ms of charge_card is all its own
slowest = overview["slowest"]
assert slowest["total"] == 9
assert slowest["items"][0]["span"]["spanId"] == ids["charge_card"]
assert slowest["items"][0]["ownTimeNano"] == 300_000_000
assert slowest["items"][0]["spanCount"] == 1
# every list item carries its direct parent, and the root has none
for name in OVERVIEW_LISTS:
for item in overview[name]["items"]:
assert item.get("parent", {}).get("spanId", "") == item["span"].get("parentSpanId", ""), f"{name}: {item}"
tree = overview["tree"]
assert tree["total"] == 9
assert tree["complete"] is True
# tree order: a span is followed by the spans below it, siblings by start time
assert [line["spanId"] for line in tree["spans"]] == [ids[name] for name in ("root", "place_order", "charge_card", "inventory_http", "inventory_grpc", "render", "consume", "model", "tool")]
place_order = tree["spans"][1]
assert (place_order["level"], place_order["childCount"], place_order["subtreeSpanCount"]) == (1, 3, 3)
assert place_order["ownTimeNano"] == 220_000_000
# a small trace carries every span's record
records = {record["spanId"]: record for record in tree["records"]}
assert set(records) == set(ids.values())
assert records[ids["charge_card"]]["attributes"]["payment.provider"] == "worldpay"
assert records[ids["charge_card"]]["statusMessage"] == "card declined"
assert records[ids["charge_card"]]["resource"]["service.name"] == orders
assert records[ids["consume"]]["links"] == [{"traceId": other_trace_id, "spanId": "aaaaaaaaaaaaaaaa"}]
assert records[ids["render"]]["events"][0]["name"] == "exception"
def test_inspect_pages_every_list(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""Every list of the overview returns 10 items and a cursor. The cursor alone returns the
rest of that list, and nothing else of the overview."""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
trace_id = TraceIdGenerator.trace_id()
root_id = TraceIdGenerator.span_id()
resources = {"service.name": "td-inspect-pages"}
spans = [Traces(timestamp=now - timedelta(seconds=30), duration=timedelta(seconds=20), trace_id=trace_id, span_id=root_id, name="POST /batch", kind=TracesKind.SPAN_KIND_SERVER, resources=resources)]
expected: dict[str, list[str]] = {name: [] for name in OVERVIEW_LISTS}
def child(group: str, i: int, offset_ms: int, **kwargs) -> None:
span_id = TraceIdGenerator.span_id()
expected[group].append(span_id)
# distinct names keep each span its own entry in the slowest list; distinct durations fix its order
spans.append(Traces(timestamp=now - timedelta(seconds=29) + timedelta(milliseconds=offset_ms), duration=timedelta(milliseconds=100 + offset_ms), trace_id=trace_id, span_id=span_id, parent_span_id=root_id, name=f"{group}-{i:02d}", resources=resources, **kwargs))
for i in range(12):
child("errors", i, i, status_code=TracesStatusCode.STATUS_CODE_ERROR, status_message=f"failure {i:02d}")
child("unmarkedFailures", i, 20 + i, attributes={"http.response.status_code": 404})
child("expensive", i, 40 + i, attributes={"gen_ai.request.model": "gpt-4o-mini", "signoz.gen_ai.usage.tokens.cost": (i + 1) / 100})
child("links", i, 60 + i, links=[TracesLink(trace_id=TraceIdGenerator.trace_id(), span_id=TraceIdGenerator.span_id(), ref_type=TracesRefType.REF_TYPE_FOLLOWS_FROM)])
insert_traces(spans)
# the slowest list ranks every span: the 48 children by duration, the longest first, then the root
by_duration = sorted(spans[1:], key=lambda span: int(span.duration_nano), reverse=True)
expected["slowest"] = [span.span_id for span in by_duration]
expected["expensive"].reverse()
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
overview = get_overview(signoz, token, trace_id)
assert overview["identity"]["spanCount"] == 49
for name in OVERVIEW_LISTS:
first = overview[name]
assert len(first["items"]) == LIST_SIZE, name
assert first["nextCursor"], name
seen = [item["span"]["spanId"] for item in first["items"]]
cursor = first["nextCursor"]
for _ in range(10):
page = get_overview(signoz, token, trace_id, {"cursor": cursor})
assert list(page) == [name], f"{name}: a page returned {list(page)}"
assert page[name]["total"] == first["total"]
seen += [item["span"]["spanId"] for item in page[name]["items"]]
cursor = page[name].get("nextCursor")
if not cursor:
break
assert len(seen) == first["total"], name
assert len(set(seen)) == len(seen), f"{name}: a span was returned twice"
if name == "slowest":
# the root's own time is what its children leave of it, so only its place is not asserted
assert [span_id for span_id in seen if span_id != root_id] == expected[name]
assert first["total"] == 49
else:
assert seen == expected[name], name
assert first["total"] == 12
# the second page of errors carries the status messages of its own spans
second = get_overview(signoz, token, trace_id, {"cursor": overview["errors"]["nextCursor"]})
assert [item["statusMessage"] for item in second["errors"]["items"]] == ["failure 10", "failure 11"]
def test_inspect_large_trace_shows_the_top_of_the_tree(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""Over 50 spans the tree holds the first 50 in time order, the records are left out, and
same-name spans under one parent are one entry of the slowest list."""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
trace_id = TraceIdGenerator.trace_id()
root_id = TraceIdGenerator.span_id()
resources = {"service.name": "td-inspect-large"}
spans = [Traces(timestamp=now - timedelta(seconds=30), duration=timedelta(seconds=20), trace_id=trace_id, span_id=root_id, name="POST /import", kind=TracesKind.SPAN_KIND_SERVER, resources=resources)]
for i in range(150):
spans.append(Traces(timestamp=now - timedelta(seconds=29) + timedelta(milliseconds=i * 10), duration=timedelta(milliseconds=5), trace_id=trace_id, span_id=TraceIdGenerator.span_id(), parent_span_id=root_id, name="write_item", resources=resources))
insert_traces(spans)
in_time_order = [span.span_id for span in spans]
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
overview = get_overview(signoz, token, trace_id)
assert overview["identity"]["spanCount"] == 151
tree = overview["tree"]
assert tree["total"] == 151
assert tree["complete"] is False
assert "records" not in tree
assert [line["spanId"] for line in tree["spans"]] == in_time_order[:50]
assert tree["spans"][0]["childCount"] == 150
slowest = overview["slowest"]
assert slowest["total"] == 2
group = next(item for item in slowest["items"] if item["span"]["name"] == "write_item")
assert group["spanCount"] == 150
assert group["ownTimeNano"] == 150 * 5_000_000
assert group["parent"]["spanId"] == root_id
assert "expensive" not in overview
def test_inspect_rejects_invalid_requests(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
trace_id = TraceIdGenerator.trace_id()
for params in (
{"depth": 3},
{"limit": 10},
{"selectFields": "http.route"},
{"cursor": "not-a-cursor"},
{"depth": "all"},
):
response = get_inspect(signoz, token, trace_id, params)
assert response.status_code == HTTPStatus.BAD_REQUEST, f"{params}: {response.text}"
missing = get_inspect(signoz, token, trace_id)
assert missing.status_code == HTTPStatus.NOT_FOUND, missing.text