mirror of
https://github.com/SigNoz/signoz.git
synced 2026-11-19 23:01:18 +00:00
Compare commits
3 Commits
test/jest-
...
issue_nerv
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
255b6fa1ac | ||
|
|
75686f4004 | ||
|
|
2df97d39e2 |
@@ -12680,6 +12680,19 @@ components:
|
||||
minimum: 0
|
||||
type: integer
|
||||
type: object
|
||||
SpantypesEventV2:
|
||||
properties:
|
||||
attributes:
|
||||
additionalProperties: {}
|
||||
type: object
|
||||
isError:
|
||||
type: boolean
|
||||
name:
|
||||
type: string
|
||||
timeUnixNano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
type: object
|
||||
SpantypesFieldContext:
|
||||
enum:
|
||||
- attribute
|
||||
@@ -12792,6 +12805,19 @@ components:
|
||||
required:
|
||||
- aggregations
|
||||
type: object
|
||||
SpantypesGettableTraceInspection:
|
||||
properties:
|
||||
find:
|
||||
$ref: '#/components/schemas/SpantypesTraceSpanList'
|
||||
overview:
|
||||
$ref: '#/components/schemas/SpantypesTraceOverview'
|
||||
span:
|
||||
$ref: '#/components/schemas/SpantypesTraceSpanDetail'
|
||||
type:
|
||||
$ref: '#/components/schemas/SpantypesInspectType'
|
||||
required:
|
||||
- type
|
||||
type: object
|
||||
SpantypesGettableTraceSummary:
|
||||
properties:
|
||||
ai:
|
||||
@@ -12872,6 +12898,19 @@ components:
|
||||
nullable: true
|
||||
type: array
|
||||
type: object
|
||||
SpantypesInspectType:
|
||||
enum:
|
||||
- overview
|
||||
- span
|
||||
- find
|
||||
type: string
|
||||
SpantypesLink:
|
||||
properties:
|
||||
spanId:
|
||||
type: string
|
||||
traceId:
|
||||
type: string
|
||||
type: object
|
||||
SpantypesOtelSpanRef:
|
||||
properties:
|
||||
refType:
|
||||
@@ -13163,6 +13202,100 @@ components:
|
||||
nullable: true
|
||||
type: object
|
||||
type: object
|
||||
SpantypesSpanRecord:
|
||||
properties:
|
||||
attributes:
|
||||
additionalProperties: {}
|
||||
type: object
|
||||
events:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesEventV2'
|
||||
type: array
|
||||
links:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesLink'
|
||||
type: array
|
||||
resource:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
spanId:
|
||||
type: string
|
||||
statusCode:
|
||||
type: string
|
||||
statusMessage:
|
||||
type: string
|
||||
truncated:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTruncatedField'
|
||||
type: array
|
||||
required:
|
||||
- spanId
|
||||
- statusCode
|
||||
- statusMessage
|
||||
- attributes
|
||||
- resource
|
||||
- events
|
||||
- links
|
||||
- truncated
|
||||
type: object
|
||||
SpantypesSpanSummary:
|
||||
properties:
|
||||
childCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
durationNano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
hasError:
|
||||
type: boolean
|
||||
kind:
|
||||
type: string
|
||||
level:
|
||||
minimum: 0
|
||||
type: integer
|
||||
name:
|
||||
type: string
|
||||
ownTimeNano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
parentSpanId:
|
||||
type: string
|
||||
responseStatusCode:
|
||||
type: string
|
||||
serviceName:
|
||||
type: string
|
||||
spanId:
|
||||
type: string
|
||||
startOffsetNano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
subtreeSpanCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- spanId
|
||||
- name
|
||||
- serviceName
|
||||
- kind
|
||||
- startOffsetNano
|
||||
- durationNano
|
||||
- ownTimeNano
|
||||
- hasError
|
||||
- level
|
||||
- childCount
|
||||
- subtreeSpanCount
|
||||
type: object
|
||||
SpantypesSpanTextMatch:
|
||||
properties:
|
||||
field:
|
||||
type: string
|
||||
snippet:
|
||||
type: string
|
||||
required:
|
||||
- field
|
||||
- snippet
|
||||
type: object
|
||||
SpantypesThreadSpan:
|
||||
properties:
|
||||
attributes:
|
||||
@@ -13229,6 +13362,23 @@ components:
|
||||
- formattedOutput
|
||||
- formatterWarnings
|
||||
type: object
|
||||
SpantypesTraceAIIdentity:
|
||||
properties:
|
||||
modelCallCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
tokens:
|
||||
$ref: '#/components/schemas/SpantypesTraceAITokens'
|
||||
toolCallCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
totalCost:
|
||||
type: number
|
||||
required:
|
||||
- modelCallCount
|
||||
- toolCallCount
|
||||
- tokens
|
||||
type: object
|
||||
SpantypesTraceAISummary:
|
||||
properties:
|
||||
tokens:
|
||||
@@ -13262,6 +13412,379 @@ components:
|
||||
- cacheWrite
|
||||
- reasoning
|
||||
type: object
|
||||
SpantypesTraceErrorItem:
|
||||
properties:
|
||||
errorAncestorCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
parent:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
span:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
statusMessage:
|
||||
type: string
|
||||
required:
|
||||
- span
|
||||
- statusMessage
|
||||
- errorAncestorCount
|
||||
type: object
|
||||
SpantypesTraceErrorList:
|
||||
properties:
|
||||
items:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTraceErrorItem'
|
||||
type: array
|
||||
nextCursor:
|
||||
type: string
|
||||
total:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- total
|
||||
- items
|
||||
type: object
|
||||
SpantypesTraceExpensiveItem:
|
||||
properties:
|
||||
cost:
|
||||
type: number
|
||||
inputTokens:
|
||||
minimum: 0
|
||||
type: integer
|
||||
outputTokens:
|
||||
minimum: 0
|
||||
type: integer
|
||||
parent:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
span:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
required:
|
||||
- span
|
||||
- inputTokens
|
||||
- outputTokens
|
||||
type: object
|
||||
SpantypesTraceExpensiveList:
|
||||
properties:
|
||||
items:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTraceExpensiveItem'
|
||||
type: array
|
||||
nextCursor:
|
||||
type: string
|
||||
total:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- total
|
||||
- items
|
||||
type: object
|
||||
SpantypesTraceIdentity:
|
||||
properties:
|
||||
ai:
|
||||
$ref: '#/components/schemas/SpantypesTraceAIIdentity'
|
||||
durationNano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
errorSpanCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
hasMissingSpans:
|
||||
type: boolean
|
||||
root:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
rootCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
serviceCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
services:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTraceServiceStats'
|
||||
type: array
|
||||
spanCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
startTimestampMillis:
|
||||
minimum: 0
|
||||
type: integer
|
||||
traceId:
|
||||
type: string
|
||||
required:
|
||||
- traceId
|
||||
- startTimestampMillis
|
||||
- durationNano
|
||||
- spanCount
|
||||
- errorSpanCount
|
||||
- rootCount
|
||||
- hasMissingSpans
|
||||
- serviceCount
|
||||
- services
|
||||
type: object
|
||||
SpantypesTraceLinkItem:
|
||||
properties:
|
||||
linkCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
links:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesLink'
|
||||
type: array
|
||||
parent:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
span:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
required:
|
||||
- span
|
||||
- linkCount
|
||||
- links
|
||||
type: object
|
||||
SpantypesTraceLinkList:
|
||||
properties:
|
||||
items:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTraceLinkItem'
|
||||
type: array
|
||||
nextCursor:
|
||||
type: string
|
||||
total:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- total
|
||||
- items
|
||||
type: object
|
||||
SpantypesTraceOverview:
|
||||
properties:
|
||||
errors:
|
||||
$ref: '#/components/schemas/SpantypesTraceErrorList'
|
||||
expensive:
|
||||
$ref: '#/components/schemas/SpantypesTraceExpensiveList'
|
||||
identity:
|
||||
$ref: '#/components/schemas/SpantypesTraceIdentity'
|
||||
links:
|
||||
$ref: '#/components/schemas/SpantypesTraceLinkList'
|
||||
slowest:
|
||||
$ref: '#/components/schemas/SpantypesTraceSlowestList'
|
||||
tree:
|
||||
$ref: '#/components/schemas/SpantypesTraceTree'
|
||||
unmarkedFailures:
|
||||
$ref: '#/components/schemas/SpantypesTraceUnmarkedFailureList'
|
||||
type: object
|
||||
SpantypesTraceServiceStats:
|
||||
properties:
|
||||
errorSpanCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
ownTimeNano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
serviceName:
|
||||
type: string
|
||||
spanCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- serviceName
|
||||
- spanCount
|
||||
- errorSpanCount
|
||||
- ownTimeNano
|
||||
type: object
|
||||
SpantypesTraceSlowestItem:
|
||||
properties:
|
||||
errorSpanCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
ownTimeNano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
parent:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
span:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
spanCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- span
|
||||
- spanCount
|
||||
- ownTimeNano
|
||||
- errorSpanCount
|
||||
type: object
|
||||
SpantypesTraceSlowestList:
|
||||
properties:
|
||||
items:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTraceSlowestItem'
|
||||
type: array
|
||||
nextCursor:
|
||||
type: string
|
||||
total:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- total
|
||||
- items
|
||||
type: object
|
||||
SpantypesTraceSpanDetail:
|
||||
properties:
|
||||
ancestorCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
ancestors:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
type: array
|
||||
descendants:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTraceSpanLevel'
|
||||
type: array
|
||||
percentile:
|
||||
$ref: '#/components/schemas/SpantypesTraceSpanPercentile'
|
||||
record:
|
||||
$ref: '#/components/schemas/SpantypesSpanRecord'
|
||||
summary:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
required:
|
||||
- summary
|
||||
- record
|
||||
- ancestors
|
||||
- ancestorCount
|
||||
- descendants
|
||||
type: object
|
||||
SpantypesTraceSpanLevel:
|
||||
properties:
|
||||
level:
|
||||
minimum: 0
|
||||
type: integer
|
||||
spans:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
type: array
|
||||
total:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- level
|
||||
- total
|
||||
- spans
|
||||
type: object
|
||||
SpantypesTraceSpanList:
|
||||
properties:
|
||||
items:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTraceSpanMatch'
|
||||
type: array
|
||||
nextCursor:
|
||||
type: string
|
||||
total:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- total
|
||||
- items
|
||||
type: object
|
||||
SpantypesTraceSpanMatch:
|
||||
properties:
|
||||
matches:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesSpanTextMatch'
|
||||
type: array
|
||||
parent:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
span:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
required:
|
||||
- span
|
||||
- matches
|
||||
type: object
|
||||
SpantypesTraceSpanPercentile:
|
||||
properties:
|
||||
p50Nano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
p90Nano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
p99Nano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
position:
|
||||
format: double
|
||||
type: number
|
||||
windowMinutes:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- p50Nano
|
||||
- p90Nano
|
||||
- p99Nano
|
||||
- position
|
||||
- windowMinutes
|
||||
type: object
|
||||
SpantypesTraceTree:
|
||||
properties:
|
||||
complete:
|
||||
type: boolean
|
||||
records:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesSpanRecord'
|
||||
type: array
|
||||
spans:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
type: array
|
||||
total:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- total
|
||||
- complete
|
||||
- spans
|
||||
type: object
|
||||
SpantypesTraceUnmarkedFailureItem:
|
||||
properties:
|
||||
parent:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
reason:
|
||||
$ref: '#/components/schemas/SpantypesUnmarkedFailureReason'
|
||||
span:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
required:
|
||||
- span
|
||||
- reason
|
||||
type: object
|
||||
SpantypesTraceUnmarkedFailureList:
|
||||
properties:
|
||||
items:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTraceUnmarkedFailureItem'
|
||||
type: array
|
||||
nextCursor:
|
||||
type: string
|
||||
total:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- total
|
||||
- items
|
||||
type: object
|
||||
SpantypesTruncatedField:
|
||||
properties:
|
||||
field:
|
||||
type: string
|
||||
length:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- field
|
||||
- length
|
||||
type: object
|
||||
SpantypesUnmarkedFailureReason:
|
||||
enum:
|
||||
- http_4xx
|
||||
- grpc_status
|
||||
- exception_event
|
||||
type: string
|
||||
SpantypesUpdatableSpanMapper:
|
||||
properties:
|
||||
config:
|
||||
@@ -19284,6 +19807,150 @@ paths:
|
||||
tags:
|
||||
- tracedetail
|
||||
x-signoz-stability: alpha
|
||||
/api/v1/traces/{traceID}/inspect:
|
||||
get:
|
||||
deprecated: false
|
||||
description: 'Returns one of three parts of a trace of any size, named by type,
|
||||
each within 50,000 characters. Without spanId or search it returns the overview:
|
||||
the trace''s identity with per-service totals, the spans where each error
|
||||
started, the failures not marked as errors (a 4xx response, a gRPC status
|
||||
other than OK, an exception event), the slowest spans by own time with same-name
|
||||
spans under one parent taken together, the most expensive gen_ai spans, and
|
||||
the span tree. The tree holds the first 50 spans of the trace: level by level
|
||||
from the root, and in time order on the level that does not fit. A trace of
|
||||
up to 50 spans also returns each span''s full record when they fit in 20,000
|
||||
characters. With spanId it returns that span''s summary and full record, its
|
||||
ancestors, and the spans below it level by level, at most 20 per level; values
|
||||
over 500 characters are cut and listed in truncated. With search it returns
|
||||
the matching spans, each with where the words were found; allSpans returns
|
||||
every span instead. Every list carries its total, and every list item the
|
||||
span''s direct parent. Pass a nextCursor as cursor with the same parameters
|
||||
to get the next page.'
|
||||
operationId: InspectTrace
|
||||
parameters:
|
||||
- description: Returns this span in full with the spans around it. Cannot be
|
||||
combined with search.
|
||||
in: query
|
||||
name: spanId
|
||||
schema:
|
||||
description: Returns this span in full with the spans around it. Cannot
|
||||
be combined with search.
|
||||
type: string
|
||||
- description: 'Returns the spans that match, earliest first: words, a filter
|
||||
expression, or both, such as worldpay AND has_error = true. Words match
|
||||
the span name, status message, attribute values and events, ignoring case.'
|
||||
in: query
|
||||
name: search
|
||||
schema:
|
||||
description: 'Returns the spans that match, earliest first: words, a filter
|
||||
expression, or both, such as worldpay AND has_error = true. Words match
|
||||
the span name, status message, attribute values and events, ignoring case.'
|
||||
type: string
|
||||
- description: With spanId, the number of levels of ancestors and of descendants
|
||||
returned, 1 to 10; 2 when not given. The root span is always returned.
|
||||
in: query
|
||||
name: depth
|
||||
schema:
|
||||
description: With spanId, the number of levels of ancestors and of descendants
|
||||
returned, 1 to 10; 2 when not given. The root span is always returned.
|
||||
type: integer
|
||||
- description: 'Returns a different response: every span of the trace as a short
|
||||
line, in tree order, or with spanId that span and every span below it. Cannot
|
||||
be combined with search or depth.'
|
||||
in: query
|
||||
name: allSpans
|
||||
schema:
|
||||
description: 'Returns a different response: every span of the trace as a
|
||||
short line, in tree order, or with spanId that span and every span below
|
||||
it. Cannot be combined with search or depth.'
|
||||
type: boolean
|
||||
- description: With spanId, attribute names returned without the 500 character
|
||||
cut; the record then holds only these attributes.
|
||||
in: query
|
||||
name: selectFields
|
||||
schema:
|
||||
description: With spanId, attribute names returned without the 500 character
|
||||
cut; the record then holds only these attributes.
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
- description: Page size for search and for allSpans, at most 1000; when not
|
||||
given, 20 with search and 400 with allSpans. A page holds fewer spans when
|
||||
it would pass the 50,000 character limit.
|
||||
in: query
|
||||
name: limit
|
||||
schema:
|
||||
description: Page size for search and for allSpans, at most 1000; when not
|
||||
given, 20 with search and 400 with allSpans. A page holds fewer spans
|
||||
when it would pass the 50,000 character limit.
|
||||
type: integer
|
||||
- description: The nextCursor of an earlier response to the same request; returns
|
||||
the next page.
|
||||
in: query
|
||||
name: cursor
|
||||
schema:
|
||||
description: The nextCursor of an earlier response to the same request;
|
||||
returns the next page.
|
||||
type: string
|
||||
- in: path
|
||||
name: traceID
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
properties:
|
||||
data:
|
||||
$ref: '#/components/schemas/SpantypesGettableTraceInspection'
|
||||
status:
|
||||
type: string
|
||||
required:
|
||||
- status
|
||||
- data
|
||||
type: object
|
||||
description: OK
|
||||
"400":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Bad Request
|
||||
"401":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Unauthorized
|
||||
"403":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Forbidden
|
||||
"404":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Not Found
|
||||
"500":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Internal Server Error
|
||||
security:
|
||||
- api_key:
|
||||
- VIEWER
|
||||
- tokenizer:
|
||||
- VIEWER
|
||||
summary: Inspect a trace
|
||||
tags:
|
||||
- tracedetail
|
||||
x-signoz-stability: alpha
|
||||
/api/v1/traces/{traceID}/summary:
|
||||
get:
|
||||
deprecated: false
|
||||
|
||||
@@ -13817,6 +13817,28 @@ export interface SpantypesEventDTO {
|
||||
timeUnixNano?: number;
|
||||
}
|
||||
|
||||
export type SpantypesEventV2DTOAttributes = { [key: string]: unknown };
|
||||
|
||||
export interface SpantypesEventV2DTO {
|
||||
/**
|
||||
* @type object
|
||||
*/
|
||||
attributes?: SpantypesEventV2DTOAttributes;
|
||||
/**
|
||||
* @type boolean
|
||||
*/
|
||||
isError?: boolean;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
name?: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
timeUnixNano?: number;
|
||||
}
|
||||
|
||||
export enum SpantypesFieldContextDTO {
|
||||
attribute = 'attribute',
|
||||
resource = 'resource',
|
||||
@@ -14136,6 +14158,168 @@ export interface SpantypesGettableTraceAggregationsDTO {
|
||||
aggregations: SpantypesSpanAggregationResultDTO[];
|
||||
}
|
||||
|
||||
export interface SpantypesSpanTextMatchDTO {
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
field: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
snippet: string;
|
||||
}
|
||||
|
||||
export interface SpantypesSpanSummaryDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
childCount: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
durationNano: number;
|
||||
/**
|
||||
* @type boolean
|
||||
*/
|
||||
hasError: boolean;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
kind: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
level: number;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
name: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
ownTimeNano: number;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
parentSpanId?: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
responseStatusCode?: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
serviceName: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
spanId: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
startOffsetNano: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
subtreeSpanCount: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceSpanMatchDTO {
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
matches: SpantypesSpanTextMatchDTO[];
|
||||
parent?: SpantypesSpanSummaryDTO;
|
||||
span: SpantypesSpanSummaryDTO;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceSpanListDTO {
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
items: SpantypesTraceSpanMatchDTO[];
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
nextCursor?: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceErrorItemDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
errorAncestorCount: number;
|
||||
parent?: SpantypesSpanSummaryDTO;
|
||||
span: SpantypesSpanSummaryDTO;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
statusMessage: string;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceErrorListDTO {
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
items: SpantypesTraceErrorItemDTO[];
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
nextCursor?: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceExpensiveItemDTO {
|
||||
/**
|
||||
* @type number
|
||||
*/
|
||||
cost?: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
inputTokens: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
outputTokens: number;
|
||||
parent?: SpantypesSpanSummaryDTO;
|
||||
span: SpantypesSpanSummaryDTO;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceExpensiveListDTO {
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
items: SpantypesTraceExpensiveItemDTO[];
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
nextCursor?: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceAITokensDTO {
|
||||
/**
|
||||
* @type integer
|
||||
@@ -14164,6 +14348,354 @@ export interface SpantypesTraceAITokensDTO {
|
||||
reasoning: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceAIIdentityDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
modelCallCount: number;
|
||||
tokens: SpantypesTraceAITokensDTO;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
toolCallCount: number;
|
||||
/**
|
||||
* @type number
|
||||
*/
|
||||
totalCost?: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceServiceStatsDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
errorSpanCount: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
ownTimeNano: number;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
serviceName: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
spanCount: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceIdentityDTO {
|
||||
ai?: SpantypesTraceAIIdentityDTO;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
durationNano: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
errorSpanCount: number;
|
||||
/**
|
||||
* @type boolean
|
||||
*/
|
||||
hasMissingSpans: boolean;
|
||||
root?: SpantypesSpanSummaryDTO;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
rootCount: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
serviceCount: number;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
services: SpantypesTraceServiceStatsDTO[];
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
spanCount: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
startTimestampMillis: number;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
traceId: string;
|
||||
}
|
||||
|
||||
export interface SpantypesLinkDTO {
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
spanId?: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
traceId?: string;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceLinkItemDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
linkCount: number;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
links: SpantypesLinkDTO[];
|
||||
parent?: SpantypesSpanSummaryDTO;
|
||||
span: SpantypesSpanSummaryDTO;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceLinkListDTO {
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
items: SpantypesTraceLinkItemDTO[];
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
nextCursor?: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceSlowestItemDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
errorSpanCount: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
ownTimeNano: number;
|
||||
parent?: SpantypesSpanSummaryDTO;
|
||||
span: SpantypesSpanSummaryDTO;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
spanCount: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceSlowestListDTO {
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
items: SpantypesTraceSlowestItemDTO[];
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
nextCursor?: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTruncatedFieldDTO {
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
field: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
length: number;
|
||||
}
|
||||
|
||||
export type SpantypesSpanRecordDTOAttributes = { [key: string]: unknown };
|
||||
|
||||
export type SpantypesSpanRecordDTOResource = { [key: string]: string };
|
||||
|
||||
export interface SpantypesSpanRecordDTO {
|
||||
/**
|
||||
* @type object
|
||||
*/
|
||||
attributes: SpantypesSpanRecordDTOAttributes;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
events: SpantypesEventV2DTO[];
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
links: SpantypesLinkDTO[];
|
||||
/**
|
||||
* @type object
|
||||
*/
|
||||
resource: SpantypesSpanRecordDTOResource;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
spanId: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
statusCode: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
statusMessage: string;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
truncated: SpantypesTruncatedFieldDTO[];
|
||||
}
|
||||
|
||||
export interface SpantypesTraceTreeDTO {
|
||||
/**
|
||||
* @type boolean
|
||||
*/
|
||||
complete: boolean;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
records?: SpantypesSpanRecordDTO[];
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
spans: SpantypesSpanSummaryDTO[];
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
total: number;
|
||||
}
|
||||
|
||||
export enum SpantypesUnmarkedFailureReasonDTO {
|
||||
http_4xx = 'http_4xx',
|
||||
grpc_status = 'grpc_status',
|
||||
exception_event = 'exception_event',
|
||||
}
|
||||
export interface SpantypesTraceUnmarkedFailureItemDTO {
|
||||
parent?: SpantypesSpanSummaryDTO;
|
||||
reason: SpantypesUnmarkedFailureReasonDTO;
|
||||
span: SpantypesSpanSummaryDTO;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceUnmarkedFailureListDTO {
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
items: SpantypesTraceUnmarkedFailureItemDTO[];
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
nextCursor?: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceOverviewDTO {
|
||||
errors?: SpantypesTraceErrorListDTO;
|
||||
expensive?: SpantypesTraceExpensiveListDTO;
|
||||
identity?: SpantypesTraceIdentityDTO;
|
||||
links?: SpantypesTraceLinkListDTO;
|
||||
slowest?: SpantypesTraceSlowestListDTO;
|
||||
tree?: SpantypesTraceTreeDTO;
|
||||
unmarkedFailures?: SpantypesTraceUnmarkedFailureListDTO;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceSpanLevelDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
level: number;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
spans: SpantypesSpanSummaryDTO[];
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceSpanPercentileDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
p50Nano: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
p90Nano: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
p99Nano: number;
|
||||
/**
|
||||
* @type number
|
||||
* @format double
|
||||
*/
|
||||
position: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
windowMinutes: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceSpanDetailDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
ancestorCount: number;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
ancestors: SpantypesSpanSummaryDTO[];
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
descendants: SpantypesTraceSpanLevelDTO[];
|
||||
percentile?: SpantypesTraceSpanPercentileDTO;
|
||||
record: SpantypesSpanRecordDTO;
|
||||
summary: SpantypesSpanSummaryDTO;
|
||||
}
|
||||
|
||||
export enum SpantypesInspectTypeDTO {
|
||||
overview = 'overview',
|
||||
span = 'span',
|
||||
find = 'find',
|
||||
}
|
||||
export interface SpantypesGettableTraceInspectionDTO {
|
||||
find?: SpantypesTraceSpanListDTO;
|
||||
overview?: SpantypesTraceOverviewDTO;
|
||||
span?: SpantypesTraceSpanDetailDTO;
|
||||
type: SpantypesInspectTypeDTO;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceAISummaryDTO {
|
||||
tokens: SpantypesTraceAITokensDTO;
|
||||
/**
|
||||
@@ -16035,6 +16567,55 @@ export type GetTraceAggregations200 = {
|
||||
status: string;
|
||||
};
|
||||
|
||||
export type InspectTracePathParameters = {
|
||||
traceID: string;
|
||||
};
|
||||
export type InspectTraceParams = {
|
||||
/**
|
||||
* @type string
|
||||
* @description Returns this span in full with the spans around it. Cannot be combined with search.
|
||||
*/
|
||||
spanId?: string;
|
||||
/**
|
||||
* @type string
|
||||
* @description Returns the spans that match, earliest first: words, a filter expression, or both, such as worldpay AND has_error = true. Words match the span name, status message, attribute values and events, ignoring case.
|
||||
*/
|
||||
search?: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @description With spanId, the number of levels of ancestors and of descendants returned, 1 to 10; 2 when not given. The root span is always returned.
|
||||
*/
|
||||
depth?: number;
|
||||
/**
|
||||
* @type boolean
|
||||
* @description Returns a different response: every span of the trace as a short line, in tree order, or with spanId that span and every span below it. Cannot be combined with search or depth.
|
||||
*/
|
||||
allSpans?: boolean;
|
||||
/**
|
||||
* @type array
|
||||
* @description With spanId, attribute names returned without the 500 character cut; the record then holds only these attributes.
|
||||
*/
|
||||
selectFields?: string[];
|
||||
/**
|
||||
* @type integer
|
||||
* @description Page size for search and for allSpans, at most 1000; when not given, 20 with search and 400 with allSpans. A page holds fewer spans when it would pass the 50,000 character limit.
|
||||
*/
|
||||
limit?: number;
|
||||
/**
|
||||
* @type string
|
||||
* @description The nextCursor of an earlier response to the same request; returns the next page.
|
||||
*/
|
||||
cursor?: string;
|
||||
};
|
||||
|
||||
export type InspectTrace200 = {
|
||||
data: SpantypesGettableTraceInspectionDTO;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
status: string;
|
||||
};
|
||||
|
||||
export type GetTraceSummaryPathParameters = {
|
||||
traceID: string;
|
||||
};
|
||||
|
||||
@@ -29,6 +29,9 @@ import type {
|
||||
GetTraceThreadPathParameters,
|
||||
GetWaterfallV4200,
|
||||
GetWaterfallV4PathParameters,
|
||||
InspectTrace200,
|
||||
InspectTraceParams,
|
||||
InspectTracePathParameters,
|
||||
RenderErrorResponseDTO,
|
||||
SpantypesPostableFlamegraphDTO,
|
||||
SpantypesPostableTraceAggregationsDTO,
|
||||
@@ -158,6 +161,117 @@ export const useGetTraceAggregations = <
|
||||
> => {
|
||||
return useMutation(getGetTraceAggregationsMutationOptions(options));
|
||||
};
|
||||
/**
|
||||
* Returns one of three parts of a trace of any size, named by type, each within 50,000 characters. Without spanId or search it returns the overview: the trace's identity with per-service totals, the spans where each error started, the failures not marked as errors (a 4xx response, a gRPC status other than OK, an exception event), the slowest spans by own time with same-name spans under one parent taken together, the most expensive gen_ai spans, and the span tree. The tree holds the first 50 spans of the trace: level by level from the root, and in time order on the level that does not fit. A trace of up to 50 spans also returns each span's full record when they fit in 20,000 characters. With spanId it returns that span's summary and full record, its ancestors, and the spans below it level by level, at most 20 per level; values over 500 characters are cut and listed in truncated. With search it returns the matching spans, each with where the words were found; allSpans returns every span instead. Every list carries its total, and every list item the span's direct parent. Pass a nextCursor as cursor with the same parameters to get the next page.
|
||||
* @summary Inspect a trace
|
||||
*/
|
||||
export const inspectTrace = (
|
||||
{ traceID }: InspectTracePathParameters,
|
||||
params?: InspectTraceParams,
|
||||
signal?: AbortSignal,
|
||||
) => {
|
||||
return GeneratedAPIInstance<InspectTrace200>({
|
||||
url: `/api/v1/traces/${traceID}/inspect`,
|
||||
method: 'GET',
|
||||
params,
|
||||
signal,
|
||||
});
|
||||
};
|
||||
|
||||
export const getInspectTraceQueryKey = (
|
||||
{ traceID }: InspectTracePathParameters,
|
||||
params?: InspectTraceParams,
|
||||
) => {
|
||||
return [
|
||||
`/api/v1/traces/${traceID}/inspect`,
|
||||
...(params ? [params] : []),
|
||||
] as const;
|
||||
};
|
||||
|
||||
export const getInspectTraceQueryOptions = <
|
||||
TData = Awaited<ReturnType<typeof inspectTrace>>,
|
||||
TError = ErrorType<RenderErrorResponseDTO>,
|
||||
>(
|
||||
{ traceID }: InspectTracePathParameters,
|
||||
params?: InspectTraceParams,
|
||||
options?: {
|
||||
query?: UseQueryOptions<
|
||||
Awaited<ReturnType<typeof inspectTrace>>,
|
||||
TError,
|
||||
TData
|
||||
>;
|
||||
},
|
||||
) => {
|
||||
const { query: queryOptions } = options ?? {};
|
||||
|
||||
const queryKey =
|
||||
queryOptions?.queryKey ?? getInspectTraceQueryKey({ traceID }, params);
|
||||
|
||||
const queryFn: QueryFunction<Awaited<ReturnType<typeof inspectTrace>>> = ({
|
||||
signal,
|
||||
}) => inspectTrace({ traceID }, params, signal);
|
||||
|
||||
return {
|
||||
queryKey,
|
||||
queryFn,
|
||||
enabled: traceID !== null && traceID !== undefined,
|
||||
...queryOptions,
|
||||
} as UseQueryOptions<
|
||||
Awaited<ReturnType<typeof inspectTrace>>,
|
||||
TError,
|
||||
TData
|
||||
> & { queryKey: QueryKey };
|
||||
};
|
||||
|
||||
export type InspectTraceQueryResult = NonNullable<
|
||||
Awaited<ReturnType<typeof inspectTrace>>
|
||||
>;
|
||||
export type InspectTraceQueryError = ErrorType<RenderErrorResponseDTO>;
|
||||
|
||||
/**
|
||||
* @summary Inspect a trace
|
||||
*/
|
||||
|
||||
export function useInspectTrace<
|
||||
TData = Awaited<ReturnType<typeof inspectTrace>>,
|
||||
TError = ErrorType<RenderErrorResponseDTO>,
|
||||
>(
|
||||
{ traceID }: InspectTracePathParameters,
|
||||
params?: InspectTraceParams,
|
||||
options?: {
|
||||
query?: UseQueryOptions<
|
||||
Awaited<ReturnType<typeof inspectTrace>>,
|
||||
TError,
|
||||
TData
|
||||
>;
|
||||
},
|
||||
): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
|
||||
const queryOptions = getInspectTraceQueryOptions({ traceID }, params, options);
|
||||
|
||||
const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
|
||||
queryKey: QueryKey;
|
||||
};
|
||||
|
||||
return withQueryKey(query, queryOptions.queryKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* @summary Inspect a trace
|
||||
*/
|
||||
export const invalidateInspectTrace = async (
|
||||
queryClient: QueryClient,
|
||||
{ traceID }: InspectTracePathParameters,
|
||||
params?: InspectTraceParams,
|
||||
options?: InvalidateOptions,
|
||||
): Promise<QueryClient> => {
|
||||
await queryClient.invalidateQueries(
|
||||
{ queryKey: getInspectTraceQueryKey({ traceID }, params) },
|
||||
options,
|
||||
);
|
||||
|
||||
return queryClient;
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns the trace's time range, root span, span and error counts, and whether any spans are missing. AI traces also include token and cost totals.
|
||||
* @summary Get summary for a trace
|
||||
|
||||
@@ -102,5 +102,23 @@ func (provider *provider) addTraceDetailRoutes(router *mux.Router) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := router.Handle("/api/v1/traces/{traceID}/inspect", handler.New(
|
||||
provider.authzMiddleware.ViewAccess(provider.traceDetailHandler.InspectTrace),
|
||||
handler.OpenAPIDef{
|
||||
ID: "InspectTrace",
|
||||
Tags: []string{"tracedetail"},
|
||||
Summary: "Inspect a trace",
|
||||
Description: "Returns one of three parts of a trace of any size, named by type, each within 50,000 characters. Without spanId or search it returns the overview: the trace's identity with per-service totals, the spans where each error started, the failures not marked as errors (a 4xx response, a gRPC status other than OK, an exception event), the slowest spans by own time with same-name spans under one parent taken together, the most expensive gen_ai spans, and the span tree. The tree holds the first 50 spans of the trace: level by level from the root, and in time order on the level that does not fit. A trace of up to 50 spans also returns each span's full record when they fit in 20,000 characters. With spanId it returns that span's summary and full record, its ancestors, and the spans below it level by level, at most 20 per level; values over 500 characters are cut and listed in truncated. With search it returns the matching spans, each with where the words were found; allSpans returns every span instead. Every list carries its total, and every list item the span's direct parent. Pass a nextCursor as cursor with the same parameters to get the next page.",
|
||||
RequestQuery: new(spantypes.InspectTraceParams),
|
||||
Response: new(spantypes.GettableTraceInspection),
|
||||
ResponseContentType: "application/json",
|
||||
SuccessStatusCode: http.StatusOK,
|
||||
ErrorStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound},
|
||||
SecuritySchemes: newSecuritySchemes(types.RoleViewer),
|
||||
},
|
||||
)).Methods(http.MethodGet).GetError(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -127,3 +127,31 @@ func (h *handler) GetThread(rw http.ResponseWriter, r *http.Request) {
|
||||
|
||||
render.Success(rw, http.StatusOK, result)
|
||||
}
|
||||
|
||||
func (h *handler) InspectTrace(rw http.ResponseWriter, r *http.Request) {
|
||||
claims, err := authtypes.ClaimsFromContext(r.Context())
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
params := new(spantypes.InspectTraceParams)
|
||||
if err := binding.Query.BindQuery(r.URL.Query(), params); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
query, err := spantypes.NewInspectQuery(params)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.module.InspectTrace(r.Context(), valuer.MustNewUUID(claims.OrgID), mux.Vars(r)["traceID"], query)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
render.Success(rw, http.StatusOK, result)
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// formattedSpansLogThreshold is the number of formatted gen_ai spans in one waterfall response above which
|
||||
@@ -292,3 +293,113 @@ func (m *module) getWindowedFlamegraph(ctx context.Context, traceID, selectedSpa
|
||||
enrichedSpans := flamegraphTrace.EnrichSelectedSpans(selectedSpans, fullSpans, selectFields)
|
||||
return spantypes.NewGettableFlamegraphTrace(enrichedSpans, start, end, true), nil
|
||||
}
|
||||
|
||||
func (m *module) InspectTrace(ctx context.Context, orgID valuer.UUID, traceID string, query *spantypes.InspectQuery) (*spantypes.GettableTraceInspection, error) {
|
||||
overview, err := m.getTraceOverview(ctx, orgID, traceID, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &spantypes.GettableTraceInspection{Type: query.Type, Overview: overview}, nil
|
||||
}
|
||||
|
||||
func (m *module) getTraceOverview(ctx context.Context, orgID valuer.UUID, traceID string, query *spantypes.InspectQuery) (*spantypes.TraceOverview, error) {
|
||||
bounds, err := m.store.GetTraceBounds(ctx, traceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A page of one list loads only what that list is built from.
|
||||
withAISpans := query.List == "" || query.List == spantypes.OverviewListExpensive
|
||||
withLinkedSpans := query.List == "" || query.List == spantypes.OverviewListLinks
|
||||
skeleton, err := m.loadSkeleton(ctx, orgID, traceID, bounds, withAISpans, withLinkedSpans)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch query.List {
|
||||
case spantypes.OverviewListErrors:
|
||||
errorList, err := m.errorPage(ctx, traceID, bounds, skeleton, query.Offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &spantypes.TraceOverview{Errors: errorList}, nil
|
||||
case spantypes.OverviewListUnmarkedFailures:
|
||||
return &spantypes.TraceOverview{UnmarkedFailures: skeleton.UnmarkedFailurePage(query.Offset)}, nil
|
||||
case spantypes.OverviewListSlowest:
|
||||
return &spantypes.TraceOverview{Slowest: skeleton.SlowestPage(query.Offset)}, nil
|
||||
case spantypes.OverviewListExpensive:
|
||||
return &spantypes.TraceOverview{Expensive: skeleton.ExpensivePage(query.Offset)}, nil
|
||||
case spantypes.OverviewListLinks:
|
||||
return &spantypes.TraceOverview{Links: skeleton.LinkPage(query.Offset)}, nil
|
||||
}
|
||||
|
||||
errorList, err := m.errorPage(ctx, traceID, bounds, skeleton, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
overview := &spantypes.TraceOverview{
|
||||
Identity: skeleton.Identity(),
|
||||
Errors: errorList,
|
||||
UnmarkedFailures: skeleton.UnmarkedFailurePage(0),
|
||||
Slowest: skeleton.SlowestPage(0),
|
||||
Expensive: skeleton.ExpensivePage(0),
|
||||
Links: skeleton.LinkPage(0),
|
||||
Tree: skeleton.Tree(),
|
||||
}
|
||||
if spantypes.IsSmallTrace(skeleton.Len()) {
|
||||
spans, err := m.store.GetTraceSpans(ctx, traceID, bounds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if records, ok := spantypes.SmallTraceRecords(spans); ok {
|
||||
overview.Tree.Records = records
|
||||
}
|
||||
}
|
||||
skeleton.FitOverview(overview)
|
||||
return overview, nil
|
||||
}
|
||||
|
||||
func (m *module) errorPage(ctx context.Context, traceID string, bounds *spantypes.TraceBounds, skeleton *spantypes.TraceSkeleton, offset int) (*spantypes.TraceErrorList, error) {
|
||||
errorList, spanIDs := skeleton.ErrorPage(offset)
|
||||
messages, err := m.store.GetSpanStatusMessages(ctx, traceID, bounds, spanIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
errorList.SetStatusMessages(messages)
|
||||
return errorList, nil
|
||||
}
|
||||
|
||||
// loadSkeleton builds the span tree of the trace, with the gen_ai usage and the span links when asked for.
|
||||
func (m *module) loadSkeleton(ctx context.Context, orgID valuer.UUID, traceID string, bounds *spantypes.TraceBounds, withAISpans, withLinkedSpans bool) (*spantypes.TraceSkeleton, error) {
|
||||
var (
|
||||
skeleton *spantypes.TraceSkeleton
|
||||
aiSpans []spantypes.AISpanUsage
|
||||
linkedSpans []spantypes.LinkedSpan
|
||||
)
|
||||
group, groupCtx := errgroup.WithContext(ctx)
|
||||
group.Go(func() error {
|
||||
var err error
|
||||
skeleton, err = m.store.GetTraceSkeleton(groupCtx, traceID, bounds)
|
||||
return err
|
||||
})
|
||||
if withAISpans {
|
||||
group.Go(func() error {
|
||||
var err error
|
||||
aiSpans, err = m.store.GetAISpanUsage(groupCtx, orgID, traceID, bounds)
|
||||
return err
|
||||
})
|
||||
}
|
||||
if withLinkedSpans {
|
||||
group.Go(func() error {
|
||||
var err error
|
||||
linkedSpans, err = m.store.GetLinkedSpans(groupCtx, traceID, bounds)
|
||||
return err
|
||||
})
|
||||
}
|
||||
if err := group.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if skeleton.Len() == 0 {
|
||||
return nil, spantypes.ErrTraceNotFound
|
||||
}
|
||||
return skeleton.WithAISpans(aiSpans).WithLinkedSpans(linkedSpans), nil
|
||||
}
|
||||
|
||||
@@ -24,6 +24,15 @@ import (
|
||||
|
||||
const colServiceName = `resource_string_service$$$$name` // $ gets escaped so $$$$ converts to $$.
|
||||
|
||||
var genAIUsageValues = []struct{ key, alias string }{
|
||||
{aiobservabilitytypes.GenAIUsageInputTokens, "input_tokens_value"},
|
||||
{aiobservabilitytypes.GenAIUsageOutputTokens, "output_tokens_value"},
|
||||
{aiobservabilitytypes.GenAIUsageCacheReadInputTokens, "cache_read_tokens_value"},
|
||||
{aiobservabilitytypes.GenAIUsageCacheCreationInputTokens, "cache_write_tokens_value"},
|
||||
{aiobservabilitytypes.GenAIUsageReasoningOutputTokens, "reasoning_tokens_value"},
|
||||
{aiobservabilitytypes.SignozGenAITotalCost, "total_cost_value"},
|
||||
}
|
||||
|
||||
// storableSpanColumns are the columns a waterfall span scans. A row's legacy maps are blanked when its
|
||||
// JSON holds the attributes.
|
||||
var storableSpanColumns = []string{
|
||||
@@ -172,20 +181,7 @@ func (s *traceStore) GetTraceStats(ctx context.Context, orgID valuer.UUID, trace
|
||||
|
||||
// genAISpanColumns returns the gen_ai columns aggregated per span, resolved across attribute evolutions.
|
||||
func (s *traceStore) genAISpanColumns(ctx context.Context, orgID valuer.UUID, bounds *spantypes.TraceBounds, sb *sqlbuilder.SelectBuilder) ([]string, error) {
|
||||
values := []struct{ key, alias string }{
|
||||
{aiobservabilitytypes.GenAIUsageInputTokens, "input_tokens_value"},
|
||||
{aiobservabilitytypes.GenAIUsageOutputTokens, "output_tokens_value"},
|
||||
{aiobservabilitytypes.GenAIUsageCacheReadInputTokens, "cache_read_tokens_value"},
|
||||
{aiobservabilitytypes.GenAIUsageCacheCreationInputTokens, "cache_write_tokens_value"},
|
||||
{aiobservabilitytypes.GenAIUsageReasoningOutputTokens, "reasoning_tokens_value"},
|
||||
{aiobservabilitytypes.SignozGenAITotalCost, "total_cost_value"},
|
||||
}
|
||||
|
||||
names := slices.Clone(aiobservabilitytypes.GenAISpanGateKeys)
|
||||
for _, value := range values {
|
||||
names = append(names, value.key)
|
||||
}
|
||||
keys, err := s.genAIFieldKeys(ctx, orgID, names)
|
||||
keys, err := s.genAIFieldKeys(ctx, orgID, genAIUsageKeyNames())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -195,9 +191,26 @@ func (s *traceStore) genAISpanColumns(ctx context.Context, orgID valuer.UUID, bo
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
columns := []string{isGenAI + " AS is_gen_ai"}
|
||||
usage, err := s.genAIUsageColumns(ctx, q, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append([]string{isGenAI + " AS is_gen_ai"}, usage...), nil
|
||||
}
|
||||
|
||||
for _, value := range values {
|
||||
// genAIUsageKeyNames lists the gate keys and the usage keys a gen_ai query resolves.
|
||||
func genAIUsageKeyNames() []string {
|
||||
names := slices.Clone(aiobservabilitytypes.GenAISpanGateKeys)
|
||||
for _, value := range genAIUsageValues {
|
||||
names = append(names, value.key)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// genAIUsageColumns returns the token and cost columns of a span, resolved across attribute evolutions.
|
||||
func (s *traceStore) genAIUsageColumns(ctx context.Context, q qbtypes.QueryInfo, keys map[string][]*telemetrytypes.TelemetryFieldKey) ([]string, error) {
|
||||
columns := make([]string, 0, len(genAIUsageValues))
|
||||
for _, value := range genAIUsageValues {
|
||||
// lookup by number, the type metadata stores numeric attributes under; float64 is only the output cast
|
||||
// fixed gen_ai keys that many traces lack, so a not-found warning means nothing here
|
||||
expr, _, err := querybuilder.ResolveColumn(ctx, q, s.storage, attributeKey(value.key, telemetrytypes.FieldDataTypeNumber), telemetrytypes.FieldDataTypeFloat64, keys)
|
||||
@@ -523,3 +536,160 @@ func (s *traceStore) anyExistsCondition(ctx context.Context, q qbtypes.QueryInfo
|
||||
}
|
||||
return sb.Or(conds...), nil
|
||||
}
|
||||
|
||||
// The collector marks an exception event with isError when it writes the span.
|
||||
const colHasExceptionEvent = `(NOT has_error AND arrayExists(e -> position(e, '"isError":true') > 0, events)) AS has_exception_event`
|
||||
|
||||
type spanStatusMessageRow struct {
|
||||
SpanID string `ch:"span_id"`
|
||||
StatusMessage string `ch:"status_message"`
|
||||
}
|
||||
|
||||
type spanLinksRow struct {
|
||||
SpanID string `ch:"span_id"`
|
||||
Links string `ch:"links"`
|
||||
}
|
||||
|
||||
func (s *traceStore) GetTraceSkeleton(ctx context.Context, traceID string, bounds *spantypes.TraceBounds) (*spantypes.TraceSkeleton, error) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(
|
||||
"DISTINCT ON (span_id) span_id", "parent_span_id", "timestamp", "duration_nano", "has_error",
|
||||
"name", colServiceName, "kind_string", "response_status_code", colHasExceptionEvent,
|
||||
)
|
||||
sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
|
||||
sb.Where(
|
||||
sb.E("trace_id", traceID),
|
||||
sb.GE("ts_bucket_start", bounds.Start.Unix()-1800),
|
||||
sb.LE("ts_bucket_start", bounds.End.Unix()),
|
||||
)
|
||||
sb.OrderByAsc("timestamp")
|
||||
sb.OrderByAsc("name")
|
||||
sb.OrderByAsc("span_id")
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
rows, err := s.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error querying trace skeleton")
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
builder := spantypes.NewTraceSkeletonBuilder(traceID, int(bounds.NumSpans))
|
||||
var span spantypes.SkeletonSpan
|
||||
// Scanning into one reused list of fields, not ScanStruct, keeps the per-row cost to the strings.
|
||||
fields := []any{
|
||||
&span.SpanID, &span.ParentSpanID, &span.StartTime, &span.DurationNano, &span.HasError,
|
||||
&span.Name, &span.ServiceName, &span.KindString, &span.ResponseStatusCode, &span.HasExceptionEvent,
|
||||
}
|
||||
for rows.Next() {
|
||||
if err := rows.Scan(fields...); err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error scanning trace skeleton")
|
||||
}
|
||||
builder.Add(&span)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error reading trace skeleton")
|
||||
}
|
||||
return builder.Build(), nil
|
||||
}
|
||||
|
||||
func (s *traceStore) GetAISpanUsage(ctx context.Context, orgID valuer.UUID, traceID string, bounds *spantypes.TraceBounds) ([]spantypes.AISpanUsage, error) {
|
||||
keys, err := s.genAIFieldKeys(ctx, orgID, genAIUsageKeyNames())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
q := querybuilder.NewQueryInfo(ctx, orgID, s.flagger, telemetrytypes.SignalTraces, nil, uint64(bounds.Start.UnixNano()), uint64(bounds.End.UnixNano()))
|
||||
// Like the waterfall, read the JSON attributes wherever that column is registered for the trace's
|
||||
// time range and the legacy maps otherwise, without waiting for the use_trace_attributes_json flag.
|
||||
q.TraceAttrsJSONOn = true
|
||||
isGenAI, err := s.anyExistsCondition(ctx, q, aiobservabilitytypes.GenAISpanGateKeys, keys, sb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
isModelCall, err := s.anyExistsCondition(ctx, q, []string{aiobservabilitytypes.GenAIRequestModel}, keys, sb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
isToolCall, err := s.anyExistsCondition(ctx, q, []string{aiobservabilitytypes.GenAIToolName}, keys, sb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
usage, err := s.genAIUsageColumns(ctx, q, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sb.Select("DISTINCT ON (span_id) span_id", isModelCall+" AS is_model_call", isToolCall+" AS is_tool_call")
|
||||
sb.SelectMore(usage...)
|
||||
sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
|
||||
sb.Where(
|
||||
sb.E("trace_id", traceID),
|
||||
sb.GE("ts_bucket_start", bounds.Start.Unix()-1800),
|
||||
sb.LE("ts_bucket_start", bounds.End.Unix()),
|
||||
isGenAI,
|
||||
)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
spans := []spantypes.AISpanUsage{}
|
||||
if err := s.telemetryStore.ClickhouseDB().Select(ctx, &spans, query, args...); err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error querying gen_ai span usage")
|
||||
}
|
||||
return spans, nil
|
||||
}
|
||||
|
||||
func (s *traceStore) GetLinkedSpans(ctx context.Context, traceID string, bounds *spantypes.TraceBounds) ([]spantypes.LinkedSpan, error) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("DISTINCT ON (span_id) span_id", "links")
|
||||
sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
|
||||
sb.Where(
|
||||
sb.E("trace_id", traceID),
|
||||
sb.GE("ts_bucket_start", bounds.Start.Unix()-1800),
|
||||
sb.LE("ts_bucket_start", bounds.End.Unix()),
|
||||
// The collector writes a link as a FOLLOWS_FROM reference; the parent is a CHILD_OF one.
|
||||
"position(links, 'FOLLOWS_FROM') > 0",
|
||||
)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
var rows []spanLinksRow
|
||||
if err := s.telemetryStore.ClickhouseDB().Select(ctx, &rows, query, args...); err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error querying linked spans")
|
||||
}
|
||||
linkedSpans := make([]spantypes.LinkedSpan, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if linked, ok := spantypes.NewLinkedSpan(row.SpanID, row.Links); ok {
|
||||
linkedSpans = append(linkedSpans, linked)
|
||||
}
|
||||
}
|
||||
return linkedSpans, nil
|
||||
}
|
||||
|
||||
func (s *traceStore) GetSpanStatusMessages(ctx context.Context, traceID string, bounds *spantypes.TraceBounds, spanIDs []string) (map[string]string, error) {
|
||||
messages := make(map[string]string, len(spanIDs))
|
||||
if len(spanIDs) == 0 {
|
||||
return messages, nil
|
||||
}
|
||||
ids := make([]any, len(spanIDs))
|
||||
for i, id := range spanIDs {
|
||||
ids[i] = id
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("DISTINCT ON (span_id) span_id", "status_message")
|
||||
sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
|
||||
sb.Where(
|
||||
sb.E("trace_id", traceID),
|
||||
sb.In("span_id", ids...),
|
||||
sb.GE("ts_bucket_start", bounds.Start.Unix()-1800),
|
||||
sb.LE("ts_bucket_start", bounds.End.Unix()),
|
||||
)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
var rows []spanStatusMessageRow
|
||||
if err := s.telemetryStore.ClickhouseDB().Select(ctx, &rows, query, args...); err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error querying span status messages")
|
||||
}
|
||||
for _, row := range rows {
|
||||
messages[row.SpanID] = row.StatusMessage
|
||||
}
|
||||
return messages, nil
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -253,3 +254,33 @@ func TestGetThreadSpans(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetLinkedSpans(t *testing.T) {
|
||||
columns := []cmock.ColumnType{{Name: "span_id", Type: "String"}, {Name: "links", Type: "String"}}
|
||||
rows := [][]any{
|
||||
{"consumer", `[{"traceId":"trace-abc123","spanId":"parent","refType":"CHILD_OF"},{"traceId":"producer-trace","spanId":"producer","refType":"FOLLOWS_FROM"}]`},
|
||||
{"parent-named-follows-from", `[{"traceId":"trace-abc123","spanId":"FOLLOWS_FROM","refType":"CHILD_OF"}]`},
|
||||
{"malformed", `not json`},
|
||||
}
|
||||
s := newTestStore(sqlmock.QueryMatcherEqual)
|
||||
s.Mock().ExpectSelect("SELECT DISTINCT ON (span_id) span_id, links FROM signoz_traces.distributed_signoz_index_v3 WHERE trace_id = ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND position(links, 'FOLLOWS_FROM') > 0").
|
||||
WillReturnRows(cmock.NewRows(columns, rows))
|
||||
|
||||
linkedSpans, err := s.Store().GetLinkedSpans(context.Background(), testTraceID, testBounds)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []spantypes.LinkedSpan{{SpanID: "consumer", Links: []spantypes.Link{{TraceID: "producer-trace", SpanID: "producer"}}}}, linkedSpans)
|
||||
assert.NoError(t, s.Mock().ExpectationsWereMet())
|
||||
}
|
||||
|
||||
// The JSON attributes are read as soon as that column is registered, with no feature flag set.
|
||||
func TestGetAISpanUsageReadsJSONAttributesWithoutFlag(t *testing.T) {
|
||||
jsonInsideTrace := testStart.Add(500 * time.Second)
|
||||
s := newTestStoreWithMetadata(sqlmock.QueryMatcherRegexp, genAIMetadataStore(&jsonInsideTrace), nil)
|
||||
s.Mock().ExpectSelect("attributes\\.`gen_ai\\.request\\.model` IS NOT NULL").WillReturnRows(cmock.NewRows(nil, nil))
|
||||
|
||||
_, err := s.Store().GetAISpanUsage(context.Background(), valuer.GenerateUUID(), testTraceID, testBounds)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.NoError(t, s.Mock().ExpectationsWereMet())
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ type Handler interface {
|
||||
GetTraceAggregations(http.ResponseWriter, *http.Request)
|
||||
GetFlamegraph(http.ResponseWriter, *http.Request)
|
||||
GetThread(http.ResponseWriter, *http.Request)
|
||||
InspectTrace(http.ResponseWriter, *http.Request)
|
||||
}
|
||||
|
||||
// Module defines the business logic for trace detail operations.
|
||||
@@ -25,4 +26,5 @@ type Module interface {
|
||||
GetTraceAggregations(ctx context.Context, traceID string, req *spantypes.PostableTraceAggregations) (*spantypes.GettableTraceAggregations, error)
|
||||
GetFlamegraph(ctx context.Context, traceID string, selectedSpanID string, selectFields []telemetrytypes.TelemetryFieldKey) (*spantypes.GettableFlamegraphTrace, error)
|
||||
GetThread(ctx context.Context, orgID valuer.UUID, traceID string, query *spantypes.ThreadQuery) (*spantypes.GettableTraceThread, error)
|
||||
InspectTrace(ctx context.Context, orgID valuer.UUID, traceID string, query *spantypes.InspectQuery) (*spantypes.GettableTraceInspection, error)
|
||||
}
|
||||
|
||||
@@ -40,6 +40,12 @@ type TraceStore interface {
|
||||
GetThreadSpans(ctx context.Context, orgID valuer.UUID, traceID string, bounds *TraceBounds, page ThreadPage) ([]StorableSpan, error)
|
||||
GetThreadCursor(ctx context.Context, traceID string, bounds *TraceBounds, spanID string) (*ThreadCursor, error)
|
||||
|
||||
GetTraceSkeleton(ctx context.Context, traceID string, bounds *TraceBounds) (*TraceSkeleton, error)
|
||||
GetAISpanUsage(ctx context.Context, orgID valuer.UUID, traceID string, bounds *TraceBounds) ([]AISpanUsage, error)
|
||||
// GetLinkedSpans returns the spans of the trace that link to other spans.
|
||||
GetLinkedSpans(ctx context.Context, traceID string, bounds *TraceBounds) ([]LinkedSpan, error)
|
||||
GetSpanStatusMessages(ctx context.Context, traceID string, bounds *TraceBounds, spanIDs []string) (map[string]string, error)
|
||||
|
||||
GetSpanCountByField(ctx context.Context, traceID string, bounds *TraceBounds, fieldKey telemetrytypes.TelemetryFieldKey) (map[string]uint64, error)
|
||||
GetSpanDurationByField(ctx context.Context, traceID string, bounds *TraceBounds, fieldKey telemetrytypes.TelemetryFieldKey) (map[string]uint64, error)
|
||||
}
|
||||
|
||||
336
pkg/types/spantypes/trace_inspect.go
Normal file
336
pkg/types/spantypes/trace_inspect.go
Normal file
@@ -0,0 +1,336 @@
|
||||
package spantypes
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
const (
|
||||
inspectListSize = 10
|
||||
inspectServiceLimit = 20
|
||||
inspectLinkTargets = 5
|
||||
// inspectTreeSpans is the number of spans the tree shows, and the largest trace returned with its records.
|
||||
inspectTreeSpans = 50
|
||||
inspectSmallTraceChars = 20_000
|
||||
// inspectMaxResponseChars keeps a response under 50,000 characters with its envelope.
|
||||
inspectMaxResponseChars = 48_000
|
||||
)
|
||||
|
||||
var (
|
||||
ErrCodeInspectInvalidCursor = errors.MustNewCode("trace_inspect_invalid_cursor")
|
||||
ErrCodeInspectInvalidDepth = errors.MustNewCode("trace_inspect_invalid_depth")
|
||||
ErrCodeInspectInvalidLimit = errors.MustNewCode("trace_inspect_invalid_limit")
|
||||
ErrCodeInspectInvalidParams = errors.MustNewCode("trace_inspect_invalid_params")
|
||||
)
|
||||
|
||||
// The lists of the overview, as a cursor names them.
|
||||
const (
|
||||
OverviewListErrors = "errors"
|
||||
OverviewListUnmarkedFailures = "unmarkedFailures"
|
||||
OverviewListSlowest = "slowest"
|
||||
OverviewListExpensive = "expensive"
|
||||
OverviewListLinks = "links"
|
||||
)
|
||||
|
||||
var overviewLists = []string{OverviewListErrors, OverviewListUnmarkedFailures, OverviewListSlowest, OverviewListExpensive, OverviewListLinks}
|
||||
|
||||
var (
|
||||
UnmarkedFailureHTTP4xx = UnmarkedFailureReason{valuer.NewString("http_4xx")}
|
||||
UnmarkedFailureGRPCStatus = UnmarkedFailureReason{valuer.NewString("grpc_status")}
|
||||
UnmarkedFailureExceptionEvent = UnmarkedFailureReason{valuer.NewString("exception_event")}
|
||||
)
|
||||
|
||||
var (
|
||||
InspectTypeOverview = InspectType{valuer.NewString("overview")}
|
||||
InspectTypeSpan = InspectType{valuer.NewString("span")}
|
||||
InspectTypeFind = InspectType{valuer.NewString("find")}
|
||||
)
|
||||
|
||||
// UnmarkedFailureReason says why a span without the error flag is listed as a failure.
|
||||
type UnmarkedFailureReason struct{ valuer.String }
|
||||
|
||||
func (UnmarkedFailureReason) Enum() []any {
|
||||
return []any{UnmarkedFailureHTTP4xx, UnmarkedFailureGRPCStatus, UnmarkedFailureExceptionEvent}
|
||||
}
|
||||
|
||||
// InspectType names the part of an inspect response that is filled.
|
||||
type InspectType struct{ valuer.String }
|
||||
|
||||
func (InspectType) Enum() []any {
|
||||
return []any{InspectTypeOverview, InspectTypeSpan, InspectTypeFind}
|
||||
}
|
||||
|
||||
type InspectTraceParams struct {
|
||||
SpanID string `query:"spanId" description:"Returns this span in full with the spans around it. Cannot be combined with search."`
|
||||
Search string `query:"search" description:"Returns the spans that match, earliest first: words, a filter expression, or both, such as worldpay AND has_error = true. Words match the span name, status message, attribute values and events, ignoring case."`
|
||||
Depth int `query:"depth" description:"With spanId, the number of levels of ancestors and of descendants returned, 1 to 10; 2 when not given. The root span is always returned."`
|
||||
AllSpans bool `query:"allSpans" description:"Returns a different response: every span of the trace as a short line, in tree order, or with spanId that span and every span below it. Cannot be combined with search or depth."`
|
||||
SelectFields []string `query:"selectFields" description:"With spanId, attribute names returned without the 500 character cut; the record then holds only these attributes."`
|
||||
Limit int `query:"limit" description:"Page size for search and for allSpans, at most 1000; when not given, 20 with search and 400 with allSpans. A page holds fewer spans when it would pass the 50,000 character limit."`
|
||||
Cursor string `query:"cursor" description:"The nextCursor of an earlier response to the same request; returns the next page."`
|
||||
}
|
||||
|
||||
// SpanSummary is the short form of a span: enough to place it in the trace and decide whether to open it.
|
||||
type SpanSummary struct {
|
||||
SpanID string `json:"spanId" required:"true"`
|
||||
ParentSpanID string `json:"parentSpanId,omitempty"`
|
||||
Name string `json:"name" required:"true"`
|
||||
ServiceName string `json:"serviceName" required:"true"`
|
||||
Kind string `json:"kind" required:"true"`
|
||||
// StartOffsetNano is the time from the start of the trace to the start of the span.
|
||||
StartOffsetNano uint64 `json:"startOffsetNano" required:"true"`
|
||||
DurationNano uint64 `json:"durationNano" required:"true"`
|
||||
// OwnTimeNano is the duration minus the time the span's children cover.
|
||||
OwnTimeNano uint64 `json:"ownTimeNano" required:"true"`
|
||||
ResponseStatusCode string `json:"responseStatusCode,omitempty"`
|
||||
HasError bool `json:"hasError" required:"true"`
|
||||
Level uint32 `json:"level" required:"true"`
|
||||
ChildCount uint32 `json:"childCount" required:"true"`
|
||||
// SubtreeSpanCount is the number of spans below this one.
|
||||
SubtreeSpanCount uint32 `json:"subtreeSpanCount" required:"true"`
|
||||
}
|
||||
|
||||
type TraceServiceStats struct {
|
||||
ServiceName string `json:"serviceName" required:"true"`
|
||||
SpanCount uint64 `json:"spanCount" required:"true"`
|
||||
ErrorSpanCount uint64 `json:"errorSpanCount" required:"true"`
|
||||
OwnTimeNano uint64 `json:"ownTimeNano" required:"true"`
|
||||
}
|
||||
|
||||
type TraceAIIdentity struct {
|
||||
ModelCallCount uint64 `json:"modelCallCount" required:"true"`
|
||||
ToolCallCount uint64 `json:"toolCallCount" required:"true"`
|
||||
Tokens TraceAITokens `json:"tokens" required:"true"`
|
||||
// TotalCost is omitted when no span carries a cost attribute.
|
||||
TotalCost *float64 `json:"totalCost,omitempty" nullable:"false"`
|
||||
}
|
||||
|
||||
type TraceIdentity struct {
|
||||
TraceID string `json:"traceId" required:"true"`
|
||||
StartTimestampMillis uint64 `json:"startTimestampMillis" required:"true"`
|
||||
DurationNano uint64 `json:"durationNano" required:"true"`
|
||||
// Root is the earliest span without a parent; omitted when every span's parent is missing.
|
||||
Root *SpanSummary `json:"root,omitempty"`
|
||||
SpanCount uint64 `json:"spanCount" required:"true"`
|
||||
ErrorSpanCount uint64 `json:"errorSpanCount" required:"true"`
|
||||
RootCount uint64 `json:"rootCount" required:"true"`
|
||||
HasMissingSpans bool `json:"hasMissingSpans" required:"true"`
|
||||
ServiceCount uint64 `json:"serviceCount" required:"true"`
|
||||
Services []TraceServiceStats `json:"services" required:"true" nullable:"false"`
|
||||
AI *TraceAIIdentity `json:"ai,omitempty"`
|
||||
}
|
||||
|
||||
// Every list has a total, items and a nextCursor when more items follow. Every item starts with
|
||||
// its span and that span's direct parent, omitted for a root or when the parent span is missing.
|
||||
type TraceErrorItem struct {
|
||||
Span SpanSummary `json:"span" required:"true"`
|
||||
Parent *SpanSummary `json:"parent,omitempty"`
|
||||
StatusMessage string `json:"statusMessage" required:"true"`
|
||||
// ErrorAncestorCount is the number of consecutive ancestors, from the parent up, that carry the error flag.
|
||||
ErrorAncestorCount uint32 `json:"errorAncestorCount" required:"true"`
|
||||
}
|
||||
|
||||
type TraceErrorList struct {
|
||||
Total uint64 `json:"total" required:"true"`
|
||||
Items []TraceErrorItem `json:"items" required:"true" nullable:"false"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
}
|
||||
|
||||
type TraceUnmarkedFailureItem struct {
|
||||
Span SpanSummary `json:"span" required:"true"`
|
||||
Parent *SpanSummary `json:"parent,omitempty"`
|
||||
Reason UnmarkedFailureReason `json:"reason" required:"true"`
|
||||
}
|
||||
|
||||
type TraceUnmarkedFailureList struct {
|
||||
Total uint64 `json:"total" required:"true"`
|
||||
Items []TraceUnmarkedFailureItem `json:"items" required:"true" nullable:"false"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
}
|
||||
|
||||
// TraceSlowestItem is one span, or the same-name spans under one parent taken together.
|
||||
type TraceSlowestItem struct {
|
||||
// Span is the longest span of the group.
|
||||
Span SpanSummary `json:"span" required:"true"`
|
||||
Parent *SpanSummary `json:"parent,omitempty"`
|
||||
SpanCount uint64 `json:"spanCount" required:"true"`
|
||||
OwnTimeNano uint64 `json:"ownTimeNano" required:"true"`
|
||||
ErrorSpanCount uint64 `json:"errorSpanCount" required:"true"`
|
||||
}
|
||||
|
||||
type TraceSlowestList struct {
|
||||
Total uint64 `json:"total" required:"true"`
|
||||
Items []TraceSlowestItem `json:"items" required:"true" nullable:"false"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
}
|
||||
|
||||
type TraceExpensiveItem struct {
|
||||
Span SpanSummary `json:"span" required:"true"`
|
||||
Parent *SpanSummary `json:"parent,omitempty"`
|
||||
Cost *float64 `json:"cost,omitempty" nullable:"false"`
|
||||
InputTokens uint64 `json:"inputTokens" required:"true"`
|
||||
OutputTokens uint64 `json:"outputTokens" required:"true"`
|
||||
}
|
||||
|
||||
type TraceExpensiveList struct {
|
||||
Total uint64 `json:"total" required:"true"`
|
||||
Items []TraceExpensiveItem `json:"items" required:"true" nullable:"false"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
}
|
||||
|
||||
type TraceLinkItem struct {
|
||||
Span SpanSummary `json:"span" required:"true"`
|
||||
Parent *SpanSummary `json:"parent,omitempty"`
|
||||
// LinkCount is the number of spans this span links to; Links holds the first of them.
|
||||
LinkCount uint64 `json:"linkCount" required:"true"`
|
||||
Links []Link `json:"links" required:"true" nullable:"false"`
|
||||
}
|
||||
|
||||
// TraceLinkList lists the spans of this trace that link to other spans, in this or another trace.
|
||||
type TraceLinkList struct {
|
||||
Total uint64 `json:"total" required:"true"`
|
||||
Items []TraceLinkItem `json:"items" required:"true" nullable:"false"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
}
|
||||
|
||||
type TraceTree struct {
|
||||
// Total is the number of spans in the trace; Spans holds every one of them when Complete is set.
|
||||
Total uint64 `json:"total" required:"true"`
|
||||
Complete bool `json:"complete" required:"true"`
|
||||
// Spans is in tree order: a span is followed by the spans below it. A span's childCount and
|
||||
// subtreeSpanCount say how much of the trace sits under it.
|
||||
Spans []SpanSummary `json:"spans" required:"true" nullable:"false"`
|
||||
// Records holds the full record of every span of a small trace.
|
||||
Records []SpanRecord `json:"records,omitempty" nullable:"false"`
|
||||
}
|
||||
|
||||
// TraceOverview holds every part on a first call and one list on a call with a cursor.
|
||||
type TraceOverview struct {
|
||||
Identity *TraceIdentity `json:"identity,omitempty"`
|
||||
Errors *TraceErrorList `json:"errors,omitempty"`
|
||||
UnmarkedFailures *TraceUnmarkedFailureList `json:"unmarkedFailures,omitempty"`
|
||||
Slowest *TraceSlowestList `json:"slowest,omitempty"`
|
||||
// Expensive is present when the trace has gen_ai spans.
|
||||
Expensive *TraceExpensiveList `json:"expensive,omitempty"`
|
||||
Links *TraceLinkList `json:"links,omitempty"`
|
||||
Tree *TraceTree `json:"tree,omitempty"`
|
||||
}
|
||||
|
||||
type TraceSpanLevel struct {
|
||||
// Level counts down from the span: 1 holds its children.
|
||||
Level uint32 `json:"level" required:"true"`
|
||||
Total uint64 `json:"total" required:"true"`
|
||||
Spans []SpanSummary `json:"spans" required:"true" nullable:"false"`
|
||||
}
|
||||
|
||||
// TraceSpanPercentile places a span's duration among the spans of the same name and service that
|
||||
// started in the window before it.
|
||||
type TraceSpanPercentile struct {
|
||||
P50Nano uint64 `json:"p50Nano" required:"true"`
|
||||
P90Nano uint64 `json:"p90Nano" required:"true"`
|
||||
P99Nano uint64 `json:"p99Nano" required:"true"`
|
||||
// Position is the share of those spans, 0 to 100, that were no slower than this one.
|
||||
Position float64 `json:"position" required:"true"`
|
||||
WindowMinutes uint64 `json:"windowMinutes" required:"true"`
|
||||
}
|
||||
|
||||
type TraceSpanDetail struct {
|
||||
Summary SpanSummary `json:"summary" required:"true"`
|
||||
Record SpanRecord `json:"record" required:"true"`
|
||||
// Percentile is omitted when it could not be computed in time.
|
||||
Percentile *TraceSpanPercentile `json:"percentile,omitempty"`
|
||||
// Ancestors holds the nearest ancestors, the parent first, then the root when it is further up.
|
||||
Ancestors []SpanSummary `json:"ancestors" required:"true" nullable:"false"`
|
||||
AncestorCount uint64 `json:"ancestorCount" required:"true"`
|
||||
Descendants []TraceSpanLevel `json:"descendants" required:"true" nullable:"false"`
|
||||
}
|
||||
|
||||
type SpanTextMatch struct {
|
||||
Field string `json:"field" required:"true"`
|
||||
Snippet string `json:"snippet" required:"true"`
|
||||
}
|
||||
|
||||
type TraceSpanMatch struct {
|
||||
Span SpanSummary `json:"span" required:"true"`
|
||||
Parent *SpanSummary `json:"parent,omitempty"`
|
||||
// Matches says where the searched words were found; empty for a filter without words.
|
||||
Matches []SpanTextMatch `json:"matches" required:"true" nullable:"false"`
|
||||
}
|
||||
|
||||
type TraceSpanList struct {
|
||||
Total uint64 `json:"total" required:"true"`
|
||||
Items []TraceSpanMatch `json:"items" required:"true" nullable:"false"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
}
|
||||
|
||||
// GettableTraceInspection holds one part, named by Type.
|
||||
type GettableTraceInspection struct {
|
||||
Type InspectType `json:"type" required:"true"`
|
||||
Overview *TraceOverview `json:"overview,omitempty"`
|
||||
Span *TraceSpanDetail `json:"span,omitempty"`
|
||||
// Find holds the spans matching search, or every span for allSpans.
|
||||
Find *TraceSpanList `json:"find,omitempty"`
|
||||
}
|
||||
|
||||
// InspectQuery is a validated inspect request.
|
||||
type InspectQuery struct {
|
||||
Type InspectType
|
||||
// List and Offset page one list of the overview; List is empty on a first call.
|
||||
List string
|
||||
Offset int
|
||||
}
|
||||
|
||||
// inspectCursor is the content of a nextCursor: the list it belongs to and where its next page starts.
|
||||
type inspectCursor struct {
|
||||
List string `json:"list,omitempty"`
|
||||
Offset int `json:"offset,omitempty"`
|
||||
}
|
||||
|
||||
// NewInspectQuery validates the parameters of an overview request. A parameter that does not
|
||||
// apply to the overview is an error.
|
||||
func NewInspectQuery(params *InspectTraceParams) (*InspectQuery, error) {
|
||||
query := &InspectQuery{Type: InspectTypeOverview}
|
||||
switch {
|
||||
case params.SpanID != "" || params.Search != "" || params.AllSpans:
|
||||
return nil, errors.New(errors.TypeUnsupported, errors.CodeUnsupported, "spanId, search and allSpans are not implemented yet")
|
||||
case params.Depth != 0:
|
||||
return nil, errors.NewInvalidInputf(ErrCodeInspectInvalidDepth, "depth needs spanId")
|
||||
case len(params.SelectFields) > 0:
|
||||
return nil, errors.NewInvalidInputf(ErrCodeInspectInvalidParams, "selectFields needs spanId")
|
||||
case params.Limit != 0:
|
||||
return nil, errors.NewInvalidInputf(ErrCodeInspectInvalidLimit, "limit needs search or allSpans")
|
||||
case params.Cursor == "":
|
||||
return query, nil
|
||||
}
|
||||
|
||||
cursor, err := decodeInspectCursor(params.Cursor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !slices.Contains(overviewLists, cursor.List) || cursor.Offset <= 0 {
|
||||
return nil, errors.NewInvalidInputf(ErrCodeInspectInvalidCursor, "invalid cursor")
|
||||
}
|
||||
query.List, query.Offset = cursor.List, cursor.Offset
|
||||
return query, nil
|
||||
}
|
||||
|
||||
func decodeInspectCursor(encoded string) (*inspectCursor, error) {
|
||||
data, err := base64.RawURLEncoding.DecodeString(encoded)
|
||||
if err != nil {
|
||||
return nil, errors.WrapInvalidInputf(err, ErrCodeInspectInvalidCursor, "invalid cursor")
|
||||
}
|
||||
cursor := new(inspectCursor)
|
||||
if err := json.Unmarshal(data, cursor); err != nil {
|
||||
return nil, errors.WrapInvalidInputf(err, ErrCodeInspectInvalidCursor, "invalid cursor")
|
||||
}
|
||||
return cursor, nil
|
||||
}
|
||||
|
||||
func (c inspectCursor) encode() string {
|
||||
data, _ := json.Marshal(c)
|
||||
return base64.RawURLEncoding.EncodeToString(data)
|
||||
}
|
||||
66
pkg/types/spantypes/trace_inspect_test.go
Normal file
66
pkg/types/spantypes/trace_inspect_test.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package spantypes
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
)
|
||||
|
||||
func TestNewInspectQuery(t *testing.T) {
|
||||
validCases := []struct {
|
||||
name string
|
||||
params InspectTraceParams
|
||||
expected InspectQuery
|
||||
}{
|
||||
{name: "NoParams_Overview", params: InspectTraceParams{}, expected: InspectQuery{Type: InspectTypeOverview}},
|
||||
{name: "ErrorsCursor_PageOfErrors", params: InspectTraceParams{Cursor: inspectCursor{List: OverviewListErrors, Offset: 10}.encode()}, expected: InspectQuery{Type: InspectTypeOverview, List: OverviewListErrors, Offset: 10}},
|
||||
{name: "SlowestCursor_PageOfSlowest", params: InspectTraceParams{Cursor: inspectCursor{List: OverviewListSlowest, Offset: 20}.encode()}, expected: InspectQuery{Type: InspectTypeOverview, List: OverviewListSlowest, Offset: 20}},
|
||||
}
|
||||
for _, testCase := range validCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
query, err := NewInspectQuery(&testCase.params)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, testCase.expected, *query)
|
||||
})
|
||||
}
|
||||
|
||||
invalidCases := []struct {
|
||||
name string
|
||||
params InspectTraceParams
|
||||
}{
|
||||
{name: "Depth_NeedsSpanID", params: InspectTraceParams{Depth: 3}},
|
||||
{name: "SelectFields_NeedsSpanID", params: InspectTraceParams{SelectFields: []string{"db.statement"}}},
|
||||
{name: "Limit_NeedsSearchOrAllSpans", params: InspectTraceParams{Limit: 10}},
|
||||
{name: "Cursor_NotBase64", params: InspectTraceParams{Cursor: "%%%"}},
|
||||
{name: "Cursor_NotJSON", params: InspectTraceParams{Cursor: "bm90IGpzb24"}},
|
||||
{name: "Cursor_UnknownList", params: InspectTraceParams{Cursor: inspectCursor{List: "tree", Offset: 10}.encode()}},
|
||||
{name: "Cursor_WithoutList", params: InspectTraceParams{Cursor: inspectCursor{Offset: 10}.encode()}},
|
||||
{name: "Cursor_WithoutOffset", params: InspectTraceParams{Cursor: inspectCursor{List: OverviewListErrors}.encode()}},
|
||||
}
|
||||
for _, testCase := range invalidCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
_, err := NewInspectQuery(&testCase.params)
|
||||
require.Error(t, err)
|
||||
assert.True(t, errors.Ast(err, errors.TypeInvalidInput))
|
||||
})
|
||||
}
|
||||
|
||||
notImplementedCases := []struct {
|
||||
name string
|
||||
params InspectTraceParams
|
||||
}{
|
||||
{name: "SpanID", params: InspectTraceParams{SpanID: "abc"}},
|
||||
{name: "Search", params: InspectTraceParams{Search: "worldpay"}},
|
||||
{name: "AllSpans", params: InspectTraceParams{AllSpans: true}},
|
||||
}
|
||||
for _, testCase := range notImplementedCases {
|
||||
t.Run(testCase.name+"_NotImplementedYet", func(t *testing.T) {
|
||||
_, err := NewInspectQuery(&testCase.params)
|
||||
require.Error(t, err)
|
||||
assert.True(t, errors.Ast(err, errors.TypeUnsupported))
|
||||
})
|
||||
}
|
||||
}
|
||||
422
pkg/types/spantypes/trace_overview.go
Normal file
422
pkg/types/spantypes/trace_overview.go
Normal file
@@ -0,0 +1,422 @@
|
||||
package spantypes
|
||||
|
||||
import (
|
||||
"container/heap"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// errorOrigins lists the error spans with no error span among their children, in start time order.
|
||||
func (s *TraceSkeleton) errorOrigins() []int32 {
|
||||
origins := make([]int32, 0)
|
||||
for i := range int32(s.Len()) {
|
||||
if !s.HasError(i) {
|
||||
continue
|
||||
}
|
||||
hasErrorChild := false
|
||||
for _, child := range s.Children(i) {
|
||||
if s.HasError(child) {
|
||||
hasErrorChild = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasErrorChild {
|
||||
origins = append(origins, i)
|
||||
}
|
||||
}
|
||||
return origins
|
||||
}
|
||||
|
||||
// ErrorPage returns one page of error origins and the ids of its spans, for their status messages.
|
||||
func (s *TraceSkeleton) ErrorPage(offset int) (*TraceErrorList, []string) {
|
||||
origins := s.errorOrigins()
|
||||
list := &TraceErrorList{Total: uint64(len(origins)), Items: []TraceErrorItem{}, NextCursor: nextListCursor(OverviewListErrors, offset, len(origins))}
|
||||
spanIDs := make([]string, 0, inspectListSize)
|
||||
for _, i := range pageOf(origins, offset, inspectListSize) {
|
||||
item := TraceErrorItem{Span: s.Summary(i), Parent: s.parentSummary(i)}
|
||||
for p := s.Parents[i]; p != noParent && s.HasError(p); p = s.Parents[p] {
|
||||
item.ErrorAncestorCount++
|
||||
}
|
||||
list.Items = append(list.Items, item)
|
||||
spanIDs = append(spanIDs, item.Span.SpanID)
|
||||
}
|
||||
return list, spanIDs
|
||||
}
|
||||
|
||||
// SetStatusMessages fills the status message of each item, cut like any other long value.
|
||||
func (list *TraceErrorList) SetStatusMessages(messages map[string]string) {
|
||||
for i := range list.Items {
|
||||
message := messages[list.Items[i].Span.SpanID]
|
||||
if cut, _, ok := cutString(message, spanRecordValueChars); ok {
|
||||
message = cut
|
||||
}
|
||||
list.Items[i].StatusMessage = message
|
||||
}
|
||||
}
|
||||
|
||||
// UnmarkedFailurePage returns one page of the spans that failed without the error flag: a 4xx
|
||||
// response, a gRPC status other than OK, or an exception event.
|
||||
func (s *TraceSkeleton) UnmarkedFailurePage(offset int) *TraceUnmarkedFailureList {
|
||||
type failure struct {
|
||||
index int32
|
||||
reason UnmarkedFailureReason
|
||||
}
|
||||
failures := make([]failure, 0)
|
||||
for i := range int32(s.Len()) {
|
||||
if s.HasError(i) {
|
||||
continue
|
||||
}
|
||||
if reason, ok := unmarkedFailureReason(s.ResponseStatusCode(i), s.HasExceptionEvent(i)); ok {
|
||||
failures = append(failures, failure{index: i, reason: reason})
|
||||
}
|
||||
}
|
||||
list := &TraceUnmarkedFailureList{Total: uint64(len(failures)), Items: []TraceUnmarkedFailureItem{}, NextCursor: nextListCursor(OverviewListUnmarkedFailures, offset, len(failures))}
|
||||
for _, f := range pageOf(failures, offset, inspectListSize) {
|
||||
list.Items = append(list.Items, TraceUnmarkedFailureItem{Span: s.Summary(f.index), Parent: s.parentSummary(f.index), Reason: f.reason})
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
// unmarkedFailureReason reads the response status column, which holds an HTTP status or a gRPC status.
|
||||
func unmarkedFailureReason(responseStatusCode string, hasExceptionEvent bool) (UnmarkedFailureReason, bool) {
|
||||
if code, err := strconv.Atoi(responseStatusCode); err == nil {
|
||||
switch {
|
||||
case code >= 400 && code < 500:
|
||||
return UnmarkedFailureHTTP4xx, true
|
||||
case code >= 1 && code <= 16:
|
||||
return UnmarkedFailureGRPCStatus, true
|
||||
}
|
||||
}
|
||||
if hasExceptionEvent {
|
||||
return UnmarkedFailureExceptionEvent, true
|
||||
}
|
||||
return UnmarkedFailureReason{}, false
|
||||
}
|
||||
|
||||
// LinkPage returns one page of the spans that link to other spans, in start time order.
|
||||
func (s *TraceSkeleton) LinkPage(offset int) *TraceLinkList {
|
||||
list := &TraceLinkList{Total: uint64(len(s.LinkedSpans)), Items: []TraceLinkItem{}, NextCursor: nextListCursor(OverviewListLinks, offset, len(s.LinkedSpans))}
|
||||
for _, linked := range pageOf(s.LinkedSpans, offset, inspectListSize) {
|
||||
idx, ok := s.Index(linked.SpanID)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
list.Items = append(list.Items, TraceLinkItem{
|
||||
Span: s.Summary(idx),
|
||||
Parent: s.parentSummary(idx),
|
||||
LinkCount: uint64(len(linked.Links)),
|
||||
Links: linked.Links[:min(len(linked.Links), inspectLinkTargets)],
|
||||
})
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
// slowestEntry is one span, or the same-name spans under one parent taken together.
|
||||
type slowestEntry struct {
|
||||
longest int32
|
||||
spanCount uint64
|
||||
ownTime uint64
|
||||
errCount uint64
|
||||
}
|
||||
|
||||
// ranksBefore orders entries by own time, the largest first, and by span order between equals,
|
||||
// so that every page cuts the same ranking.
|
||||
func (e slowestEntry) ranksBefore(other slowestEntry) bool {
|
||||
if e.ownTime != other.ownTime {
|
||||
return e.ownTime > other.ownTime
|
||||
}
|
||||
return e.longest < other.longest
|
||||
}
|
||||
|
||||
// slowestHeap holds the best-ranked entries seen so far, the last of them on top.
|
||||
type slowestHeap []slowestEntry
|
||||
|
||||
func (h slowestHeap) Len() int { return len(h) }
|
||||
func (h slowestHeap) Less(i, j int) bool { return h[j].ranksBefore(h[i]) }
|
||||
func (h slowestHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
|
||||
func (h *slowestHeap) Push(x any) { *h = append(*h, x.(slowestEntry)) }
|
||||
func (h *slowestHeap) Pop() any {
|
||||
old := *h
|
||||
last := old[len(old)-1]
|
||||
*h = old[:len(old)-1]
|
||||
return last
|
||||
}
|
||||
|
||||
// SlowestPage returns one page of the ranking by own time, same-name spans under one parent
|
||||
// counted as one entry. Only the entries up to the end of the page are kept while ranking.
|
||||
func (s *TraceSkeleton) SlowestPage(offset int) *TraceSlowestList {
|
||||
keep := offset + inspectListSize
|
||||
top := &slowestHeap{}
|
||||
total := 0
|
||||
offer := func(e slowestEntry) {
|
||||
total++
|
||||
switch {
|
||||
case top.Len() < keep:
|
||||
heap.Push(top, e)
|
||||
case e.ranksBefore((*top)[0]):
|
||||
(*top)[0] = e
|
||||
heap.Fix(top, 0)
|
||||
}
|
||||
}
|
||||
single := func(i int32) slowestEntry {
|
||||
e := slowestEntry{longest: i, spanCount: 1, ownTime: s.ownTime[i]}
|
||||
if s.HasError(i) {
|
||||
e.errCount = 1
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
for _, root := range s.roots {
|
||||
offer(single(root))
|
||||
}
|
||||
var siblings []int32
|
||||
for parent := range int32(s.Len()) {
|
||||
children := s.Children(parent)
|
||||
if len(children) == 1 {
|
||||
offer(single(children[0]))
|
||||
continue
|
||||
}
|
||||
siblings = append(siblings[:0], children...)
|
||||
slices.SortStableFunc(siblings, func(a, b int32) int {
|
||||
if s.Names[a] != s.Names[b] {
|
||||
return int(s.Names[a]) - int(s.Names[b])
|
||||
}
|
||||
return int(s.Services[a]) - int(s.Services[b])
|
||||
})
|
||||
for k := 0; k < len(siblings); {
|
||||
e := single(siblings[k])
|
||||
next := k + 1
|
||||
for ; next < len(siblings) && s.Names[siblings[next]] == s.Names[siblings[k]] && s.Services[siblings[next]] == s.Services[siblings[k]]; next++ {
|
||||
member := single(siblings[next])
|
||||
e.spanCount++
|
||||
e.ownTime += member.ownTime
|
||||
e.errCount += member.errCount
|
||||
if s.DurationNano[siblings[next]] > s.DurationNano[e.longest] {
|
||||
e.longest = siblings[next]
|
||||
}
|
||||
}
|
||||
offer(e)
|
||||
k = next
|
||||
}
|
||||
}
|
||||
|
||||
ranked := []slowestEntry(*top)
|
||||
slices.SortFunc(ranked, func(a, b slowestEntry) int {
|
||||
if a.ranksBefore(b) {
|
||||
return -1
|
||||
}
|
||||
return 1
|
||||
})
|
||||
list := &TraceSlowestList{Total: uint64(total), Items: []TraceSlowestItem{}, NextCursor: nextListCursor(OverviewListSlowest, offset, total)}
|
||||
for _, e := range pageOf(ranked, offset, inspectListSize) {
|
||||
list.Items = append(list.Items, TraceSlowestItem{Span: s.Summary(e.longest), Parent: s.parentSummary(e.longest), SpanCount: e.spanCount, OwnTimeNano: e.ownTime, ErrorSpanCount: e.errCount})
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
// ExpensivePage returns one page of the gen_ai spans ranked by cost, or by tokens when no span
|
||||
// carries a cost; nil without gen_ai spans.
|
||||
func (s *TraceSkeleton) ExpensivePage(offset int) *TraceExpensiveList {
|
||||
if len(s.AISpans) == 0 {
|
||||
return nil
|
||||
}
|
||||
hasCost := false
|
||||
for i := range s.AISpans {
|
||||
if s.AISpans[i].Cost != nil {
|
||||
hasCost = true
|
||||
break
|
||||
}
|
||||
}
|
||||
type rankedSpan struct {
|
||||
index int32
|
||||
weight float64
|
||||
usage *AISpanUsage
|
||||
}
|
||||
ranked := make([]rankedSpan, 0, len(s.AISpans))
|
||||
for i := range s.AISpans {
|
||||
usage := &s.AISpans[i]
|
||||
index, ok := s.Index(usage.SpanID)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
weight := floatValue(usage.InputTokens) + floatValue(usage.OutputTokens)
|
||||
if hasCost {
|
||||
weight = floatValue(usage.Cost)
|
||||
}
|
||||
ranked = append(ranked, rankedSpan{index: index, weight: weight, usage: usage})
|
||||
}
|
||||
// Equal weights fall back to span order, so that every page cuts the same ranking.
|
||||
slices.SortFunc(ranked, func(a, b rankedSpan) int {
|
||||
switch {
|
||||
case a.weight > b.weight:
|
||||
return -1
|
||||
case a.weight < b.weight:
|
||||
return 1
|
||||
}
|
||||
return int(a.index) - int(b.index)
|
||||
})
|
||||
|
||||
list := &TraceExpensiveList{Total: uint64(len(ranked)), Items: []TraceExpensiveItem{}, NextCursor: nextListCursor(OverviewListExpensive, offset, len(ranked))}
|
||||
for _, span := range pageOf(ranked, offset, inspectListSize) {
|
||||
list.Items = append(list.Items, TraceExpensiveItem{
|
||||
Span: s.Summary(span.index),
|
||||
Parent: s.parentSummary(span.index),
|
||||
Cost: span.usage.Cost,
|
||||
InputTokens: uint64(floatValue(span.usage.InputTokens)),
|
||||
OutputTokens: uint64(floatValue(span.usage.OutputTokens)),
|
||||
})
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) Identity() *TraceIdentity {
|
||||
identity := &TraceIdentity{
|
||||
TraceID: s.TraceID,
|
||||
StartTimestampMillis: uint64(s.startNs / 1_000_000),
|
||||
DurationNano: uint64(s.endNs - s.startNs),
|
||||
SpanCount: uint64(s.Len()),
|
||||
ErrorSpanCount: s.errCount,
|
||||
RootCount: uint64(len(s.roots)),
|
||||
HasMissingSpans: s.HasMissingSpans(),
|
||||
}
|
||||
if root, ok := s.PrimaryRoot(); ok && !s.HasMissingParent(root) {
|
||||
summary := s.Summary(root)
|
||||
identity.Root = &summary
|
||||
}
|
||||
|
||||
stats := make([]TraceServiceStats, len(s.Strings))
|
||||
for i := range int32(s.Len()) {
|
||||
service := &stats[s.Services[i]]
|
||||
service.SpanCount++
|
||||
service.OwnTimeNano += s.ownTime[i]
|
||||
if s.HasError(i) {
|
||||
service.ErrorSpanCount++
|
||||
}
|
||||
}
|
||||
services := make([]TraceServiceStats, 0)
|
||||
for idx := range stats {
|
||||
if stats[idx].SpanCount > 0 {
|
||||
stats[idx].ServiceName = s.Strings[idx]
|
||||
services = append(services, stats[idx])
|
||||
}
|
||||
}
|
||||
slices.SortStableFunc(services, func(a, b TraceServiceStats) int {
|
||||
switch {
|
||||
case a.OwnTimeNano > b.OwnTimeNano:
|
||||
return -1
|
||||
case a.OwnTimeNano < b.OwnTimeNano:
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
identity.ServiceCount = uint64(len(services))
|
||||
identity.Services = services[:min(len(services), inspectServiceLimit)]
|
||||
|
||||
if len(s.AISpans) > 0 {
|
||||
ai := &TraceAIIdentity{}
|
||||
for i := range s.AISpans {
|
||||
usage := &s.AISpans[i]
|
||||
if usage.IsModelCall {
|
||||
ai.ModelCallCount++
|
||||
}
|
||||
if usage.IsToolCall {
|
||||
ai.ToolCallCount++
|
||||
}
|
||||
ai.Tokens.Input += uint64(floatValue(usage.InputTokens))
|
||||
ai.Tokens.Output += uint64(floatValue(usage.OutputTokens))
|
||||
ai.Tokens.CacheRead += uint64(floatValue(usage.CacheReadTokens))
|
||||
ai.Tokens.CacheWrite += uint64(floatValue(usage.CacheWriteTokens))
|
||||
ai.Tokens.Reasoning += uint64(floatValue(usage.ReasoningTokens))
|
||||
if usage.Cost != nil {
|
||||
if ai.TotalCost == nil {
|
||||
ai.TotalCost = new(float64)
|
||||
}
|
||||
*ai.TotalCost += *usage.Cost
|
||||
}
|
||||
}
|
||||
identity.AI = ai
|
||||
}
|
||||
return identity
|
||||
}
|
||||
|
||||
// Tree lists the first 50 spans of the trace: level by level from the roots, and in time order on
|
||||
// the level that does not fit. It is complete when the trace has no more spans than that.
|
||||
func (s *TraceSkeleton) Tree() *TraceTree {
|
||||
return s.tree(inspectTreeSpans)
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) tree(limit int) *TraceTree {
|
||||
tree := &TraceTree{Total: uint64(s.Len()), Complete: s.Len() <= limit}
|
||||
shown := make(map[int32]struct{}, min(limit, s.Len()))
|
||||
for level := s.roots; len(level) > 0 && len(shown) < limit; {
|
||||
level = level[:min(len(level), limit-len(shown))]
|
||||
next := make([]int32, 0)
|
||||
for _, i := range level {
|
||||
shown[i] = struct{}{}
|
||||
next = append(next, s.Children(i)...)
|
||||
}
|
||||
level = next
|
||||
}
|
||||
|
||||
// The shown spans are returned in tree order: a span is followed by the spans below it.
|
||||
tree.Spans = make([]SpanSummary, 0, len(shown))
|
||||
for _, i := range s.preorder {
|
||||
if _, ok := shown[i]; ok {
|
||||
tree.Spans = append(tree.Spans, s.Summary(i))
|
||||
}
|
||||
}
|
||||
return tree
|
||||
}
|
||||
|
||||
// FitOverview drops the span records and then shrinks the tree until the overview fits the
|
||||
// response size limit.
|
||||
func (s *TraceSkeleton) FitOverview(overview *TraceOverview) {
|
||||
for limit := len(overview.Tree.Spans); jsonChars(overview) > inspectMaxResponseChars && limit > 0; {
|
||||
if overview.Tree.Records != nil {
|
||||
overview.Tree.Records = nil
|
||||
continue
|
||||
}
|
||||
over := jsonChars(overview) - inspectMaxResponseChars
|
||||
perSpan := jsonChars(overview.Tree.Spans)/len(overview.Tree.Spans) + 1
|
||||
limit = max(limit-over/perSpan-1, 0)
|
||||
overview.Tree = s.tree(limit)
|
||||
}
|
||||
}
|
||||
|
||||
// parentSummary returns the short form of span i's direct parent; nil for a root or a missing parent.
|
||||
func (s *TraceSkeleton) parentSummary(i int32) *SpanSummary {
|
||||
parent := s.Parents[i]
|
||||
if parent == noParent {
|
||||
return nil
|
||||
}
|
||||
summary := s.Summary(parent)
|
||||
return &summary
|
||||
}
|
||||
|
||||
// nextListCursor returns the cursor of the page after the one starting at offset; empty on the last page.
|
||||
func nextListCursor(list string, offset, total int) string {
|
||||
if offset+inspectListSize >= total {
|
||||
return ""
|
||||
}
|
||||
return inspectCursor{List: list, Offset: offset + inspectListSize}.encode()
|
||||
}
|
||||
|
||||
func pageOf[T any](items []T, offset, limit int) []T {
|
||||
if offset >= len(items) {
|
||||
return nil
|
||||
}
|
||||
return items[offset:min(offset+limit, len(items))]
|
||||
}
|
||||
|
||||
func jsonChars(value any) int {
|
||||
data, _ := json.Marshal(value)
|
||||
return len(data)
|
||||
}
|
||||
|
||||
func floatValue(value *float64) float64 {
|
||||
if value == nil {
|
||||
return 0
|
||||
}
|
||||
return *value
|
||||
}
|
||||
484
pkg/types/spantypes/trace_overview_test.go
Normal file
484
pkg/types/spantypes/trace_overview_test.go
Normal file
@@ -0,0 +1,484 @@
|
||||
package spantypes
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func summaryIDs(summaries []SpanSummary) []string {
|
||||
ids := make([]string, 0, len(summaries))
|
||||
for _, summary := range summaries {
|
||||
ids = append(ids, summary.SpanID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// nextOffset reads the offset out of a list's nextCursor, as a request carrying it would.
|
||||
func nextOffset(t *testing.T, list, cursor string) int {
|
||||
t.Helper()
|
||||
query, err := NewInspectQuery(&InspectTraceParams{Cursor: cursor})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, list, query.List)
|
||||
return query.Offset
|
||||
}
|
||||
|
||||
func TestErrorPage(t *testing.T) {
|
||||
t.Run("ErrorCarriedUpToRoot_ListsWhereItStarted", func(t *testing.T) {
|
||||
specs := []skeletonSpec{{id: "level-0", startMs: 0, durationMs: 100, hasError: true}}
|
||||
for depth := 1; depth <= 12; depth++ {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("level-%d", depth), parent: fmt.Sprintf("level-%d", depth-1), startMs: int64(depth), durationMs: 50, hasError: true})
|
||||
}
|
||||
specs = append(specs, skeletonSpec{id: "healthy", parent: "level-0", startMs: 60, durationMs: 5})
|
||||
skeleton := buildSkeleton(specs...)
|
||||
|
||||
list, spanIDs := skeleton.ErrorPage(0)
|
||||
|
||||
assert.Equal(t, uint64(1), list.Total)
|
||||
assert.Equal(t, []string{"level-12"}, spanIDs)
|
||||
require.Len(t, list.Items, 1)
|
||||
assert.Equal(t, uint32(12), list.Items[0].ErrorAncestorCount)
|
||||
require.NotNil(t, list.Items[0].Parent)
|
||||
assert.Equal(t, "level-11", list.Items[0].Parent.SpanID)
|
||||
assert.Empty(t, list.NextCursor)
|
||||
})
|
||||
|
||||
t.Run("ParentFailsOnItsOwn_ListsChildOnly", func(t *testing.T) {
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", startMs: 0, durationMs: 100},
|
||||
skeletonSpec{id: "parent", parent: "root", startMs: 1, durationMs: 50, hasError: true},
|
||||
skeletonSpec{id: "child", parent: "parent", startMs: 2, durationMs: 10, hasError: true},
|
||||
skeletonSpec{id: "other", parent: "root", startMs: 60, durationMs: 10, hasError: true},
|
||||
)
|
||||
|
||||
list, spanIDs := skeleton.ErrorPage(0)
|
||||
|
||||
assert.Equal(t, uint64(2), list.Total)
|
||||
assert.Equal(t, []string{"child", "other"}, spanIDs)
|
||||
})
|
||||
|
||||
t.Run("MoreThanOnePage_PagesByCursorInStartOrder", func(t *testing.T) {
|
||||
specs := []skeletonSpec{{id: "root", startMs: 0, durationMs: 1000}}
|
||||
for i := range 23 {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("err-%02d", i), parent: "root", startMs: int64(i + 1), durationMs: 1, hasError: true})
|
||||
}
|
||||
skeleton := buildSkeleton(specs...)
|
||||
|
||||
seen := make([]string, 0, 23)
|
||||
offset := 0
|
||||
for range 3 {
|
||||
list, spanIDs := skeleton.ErrorPage(offset)
|
||||
assert.Equal(t, uint64(23), list.Total)
|
||||
seen = append(seen, spanIDs...)
|
||||
if list.NextCursor == "" {
|
||||
break
|
||||
}
|
||||
offset = nextOffset(t, OverviewListErrors, list.NextCursor)
|
||||
}
|
||||
|
||||
require.Len(t, seen, 23)
|
||||
assert.Equal(t, "err-00", seen[0])
|
||||
assert.Equal(t, "err-10", seen[10])
|
||||
assert.Equal(t, "err-22", seen[22])
|
||||
})
|
||||
|
||||
t.Run("RootError_HasNoParentAndItsMessageIsCut", func(t *testing.T) {
|
||||
skeleton := buildSkeleton(skeletonSpec{id: "root", startMs: 0, durationMs: 10, hasError: true})
|
||||
list, _ := skeleton.ErrorPage(0)
|
||||
message := ""
|
||||
for range 60 {
|
||||
message += "0123456789"
|
||||
}
|
||||
|
||||
list.SetStatusMessages(map[string]string{"root": message})
|
||||
|
||||
assert.Nil(t, list.Items[0].Parent)
|
||||
assert.Len(t, []rune(list.Items[0].StatusMessage), spanRecordValueChars+1)
|
||||
})
|
||||
}
|
||||
|
||||
func TestUnmarkedFailurePage(t *testing.T) {
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", startMs: 0, durationMs: 100, statusCode: "200"},
|
||||
skeletonSpec{id: "not-found", parent: "root", startMs: 1, durationMs: 1, statusCode: "404"},
|
||||
skeletonSpec{id: "flagged-4xx", parent: "root", startMs: 2, durationMs: 1, statusCode: "409", hasError: true},
|
||||
skeletonSpec{id: "grpc-ok", parent: "root", startMs: 3, durationMs: 1, statusCode: "0"},
|
||||
skeletonSpec{id: "grpc-not-found", parent: "root", startMs: 4, durationMs: 1, statusCode: "5"},
|
||||
skeletonSpec{id: "handled-exception", parent: "root", startMs: 5, durationMs: 1, hasException: true},
|
||||
skeletonSpec{id: "server-error", parent: "root", startMs: 6, durationMs: 1, statusCode: "503"},
|
||||
skeletonSpec{id: "json-rpc", parent: "root", startMs: 7, durationMs: 1, statusCode: "-32601"},
|
||||
)
|
||||
|
||||
list := skeleton.UnmarkedFailurePage(0)
|
||||
|
||||
require.Len(t, list.Items, 3)
|
||||
assert.Equal(t, uint64(3), list.Total)
|
||||
assert.Equal(t, "not-found", list.Items[0].Span.SpanID)
|
||||
assert.Equal(t, UnmarkedFailureHTTP4xx, list.Items[0].Reason)
|
||||
require.NotNil(t, list.Items[0].Parent)
|
||||
assert.Equal(t, "root", list.Items[0].Parent.SpanID)
|
||||
assert.Equal(t, "grpc-not-found", list.Items[1].Span.SpanID)
|
||||
assert.Equal(t, UnmarkedFailureGRPCStatus, list.Items[1].Reason)
|
||||
assert.Equal(t, "handled-exception", list.Items[2].Span.SpanID)
|
||||
assert.Equal(t, UnmarkedFailureExceptionEvent, list.Items[2].Reason)
|
||||
}
|
||||
|
||||
func TestLinkPage(t *testing.T) {
|
||||
specs := []skeletonSpec{{id: "root", startMs: 0, durationMs: 1000}}
|
||||
linkedSpans := make([]LinkedSpan, 0, 14)
|
||||
for i := range 13 {
|
||||
id := fmt.Sprintf("consumer-%02d", i)
|
||||
specs = append(specs, skeletonSpec{id: id, parent: "root", startMs: int64(i + 1), durationMs: 1})
|
||||
// Given in reverse so the page order has to come from the spans, not from the input.
|
||||
linkedSpans = append([]LinkedSpan{{SpanID: id, Links: []Link{{TraceID: "producer-trace", SpanID: id + "-producer"}}}}, linkedSpans...)
|
||||
}
|
||||
batch := LinkedSpan{SpanID: "consumer-00"}
|
||||
for i := range 8 {
|
||||
batch.Links = append(batch.Links, Link{TraceID: fmt.Sprintf("producer-trace-%d", i), SpanID: "producer"})
|
||||
}
|
||||
linkedSpans[len(linkedSpans)-1] = batch
|
||||
linkedSpans = append(linkedSpans, LinkedSpan{SpanID: "not-loaded", Links: []Link{{TraceID: "t", SpanID: "s"}}})
|
||||
skeleton := buildSkeleton(specs...).WithLinkedSpans(linkedSpans)
|
||||
|
||||
first := skeleton.LinkPage(0)
|
||||
|
||||
assert.Equal(t, uint64(13), first.Total)
|
||||
require.Len(t, first.Items, inspectListSize)
|
||||
assert.Equal(t, "consumer-00", first.Items[0].Span.SpanID)
|
||||
require.NotNil(t, first.Items[0].Parent)
|
||||
assert.Equal(t, "root", first.Items[0].Parent.SpanID)
|
||||
assert.Equal(t, uint64(8), first.Items[0].LinkCount)
|
||||
assert.Len(t, first.Items[0].Links, inspectLinkTargets)
|
||||
assert.Equal(t, "consumer-09", first.Items[9].Span.SpanID)
|
||||
|
||||
second := skeleton.LinkPage(nextOffset(t, OverviewListLinks, first.NextCursor))
|
||||
require.Len(t, second.Items, 3)
|
||||
assert.Equal(t, "consumer-12", second.Items[2].Span.SpanID)
|
||||
assert.Empty(t, second.NextCursor)
|
||||
|
||||
empty := buildSkeleton(skeletonSpec{id: "root", startMs: 0, durationMs: 1}).LinkPage(0)
|
||||
assert.Equal(t, uint64(0), empty.Total)
|
||||
assert.Empty(t, empty.Items)
|
||||
}
|
||||
|
||||
func TestSlowestPage(t *testing.T) {
|
||||
t.Run("SameNameUnderOneParent_CountsAsOneEntry", func(t *testing.T) {
|
||||
specs := []skeletonSpec{
|
||||
{id: "root", name: "POST /graphql", service: "api", startMs: 0, durationMs: 10_000},
|
||||
{id: "resolve", parent: "root", name: "resolve_plans", service: "api", startMs: 1, durationMs: 9_000},
|
||||
{id: "slow-single", parent: "root", name: "render", service: "api", startMs: 9_100, durationMs: 300},
|
||||
}
|
||||
for i := range 416 {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("cost-%03d", i), parent: "resolve", name: "calculate_annual_cost", service: "pricing", startMs: int64(2 + i*20), durationMs: int64(10 + i%3), hasError: i < 3})
|
||||
}
|
||||
skeleton := buildSkeleton(specs...)
|
||||
|
||||
list := skeleton.SlowestPage(0)
|
||||
|
||||
require.NotEmpty(t, list.Items)
|
||||
assert.Equal(t, uint64(4), list.Total)
|
||||
assert.Empty(t, list.NextCursor)
|
||||
group := list.Items[0]
|
||||
assert.Equal(t, "calculate_annual_cost", group.Span.Name)
|
||||
assert.Equal(t, uint64(416), group.SpanCount)
|
||||
assert.Equal(t, uint64(3), group.ErrorSpanCount)
|
||||
assert.Equal(t, uint64(12*time.Millisecond), group.Span.DurationNano)
|
||||
require.NotNil(t, group.Parent)
|
||||
assert.Equal(t, "resolve", group.Parent.SpanID)
|
||||
assert.Equal(t, "resolve", list.Items[1].Span.SpanID)
|
||||
assert.Equal(t, uint64(1), list.Items[1].SpanCount)
|
||||
})
|
||||
|
||||
t.Run("SameNameUnderDifferentParents_StaysSeparate", func(t *testing.T) {
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", name: "root", service: "api", startMs: 0, durationMs: 107},
|
||||
skeletonSpec{id: "left", parent: "root", name: "branch", service: "api", startMs: 0, durationMs: 40},
|
||||
skeletonSpec{id: "left-query", parent: "left", name: "SELECT", service: "db", startMs: 1, durationMs: 30},
|
||||
skeletonSpec{id: "right", parent: "root", name: "branch", service: "worker", startMs: 50, durationMs: 40},
|
||||
skeletonSpec{id: "right-query", parent: "right", name: "SELECT", service: "db", startMs: 51, durationMs: 15},
|
||||
)
|
||||
|
||||
list := skeleton.SlowestPage(0)
|
||||
|
||||
assert.Equal(t, uint64(5), list.Total)
|
||||
assert.Equal(t, []string{"left-query", "root", "right", "right-query", "left"}, []string{
|
||||
list.Items[0].Span.SpanID, list.Items[1].Span.SpanID, list.Items[2].Span.SpanID, list.Items[3].Span.SpanID, list.Items[4].Span.SpanID,
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("MoreThanOnePage_PagesOneRankingByCursor", func(t *testing.T) {
|
||||
specs := []skeletonSpec{{id: "root", name: "root", service: "api", startMs: 0, durationMs: 1}}
|
||||
for i := range 30 {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("op-%02d", i), parent: "root", name: fmt.Sprintf("op-%02d", i), service: "api", startMs: 10, durationMs: int64(100 + i)})
|
||||
}
|
||||
skeleton := buildSkeleton(specs...)
|
||||
|
||||
seen := make([]string, 0, 31)
|
||||
offset := 0
|
||||
for range 5 {
|
||||
list := skeleton.SlowestPage(offset)
|
||||
assert.Equal(t, uint64(31), list.Total)
|
||||
for _, item := range list.Items {
|
||||
seen = append(seen, item.Span.SpanID)
|
||||
}
|
||||
if list.NextCursor == "" {
|
||||
break
|
||||
}
|
||||
offset = nextOffset(t, OverviewListSlowest, list.NextCursor)
|
||||
}
|
||||
|
||||
require.Len(t, seen, 31)
|
||||
assert.Equal(t, "op-29", seen[0])
|
||||
assert.Equal(t, "op-20", seen[9])
|
||||
assert.Equal(t, "op-19", seen[10])
|
||||
assert.Equal(t, "op-00", seen[29])
|
||||
assert.Equal(t, "root", seen[30])
|
||||
})
|
||||
|
||||
t.Run("EqualOwnTime_PagesInSpanOrder", func(t *testing.T) {
|
||||
specs := []skeletonSpec{{id: "root", name: "root", service: "api", startMs: 0, durationMs: 1}}
|
||||
for i := range 25 {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("op-%02d", i), parent: "root", name: fmt.Sprintf("op-%02d", i), service: "api", startMs: int64(10 + i), durationMs: 5})
|
||||
}
|
||||
skeleton := buildSkeleton(specs...)
|
||||
|
||||
first, second, third := skeleton.SlowestPage(0), skeleton.SlowestPage(10), skeleton.SlowestPage(20)
|
||||
|
||||
assert.Equal(t, "op-00", first.Items[0].Span.SpanID)
|
||||
assert.Equal(t, "op-09", first.Items[9].Span.SpanID)
|
||||
assert.Equal(t, "op-10", second.Items[0].Span.SpanID)
|
||||
assert.Equal(t, "op-19", second.Items[9].Span.SpanID)
|
||||
require.Len(t, third.Items, 6)
|
||||
assert.Equal(t, "op-24", third.Items[4].Span.SpanID)
|
||||
assert.Equal(t, "root", third.Items[5].Span.SpanID)
|
||||
assert.Empty(t, skeleton.SlowestPage(40).Items)
|
||||
})
|
||||
}
|
||||
|
||||
func TestTree(t *testing.T) {
|
||||
t.Run("TraceWithinLimit_ListsEverySpanInTreeOrder", func(t *testing.T) {
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", startMs: 0, durationMs: 100},
|
||||
skeletonSpec{id: "a", parent: "root", startMs: 1, durationMs: 40},
|
||||
skeletonSpec{id: "b", parent: "root", startMs: 50, durationMs: 40},
|
||||
skeletonSpec{id: "a-child", parent: "a", startMs: 60, durationMs: 1},
|
||||
)
|
||||
|
||||
tree := skeleton.tree(4)
|
||||
|
||||
assert.True(t, tree.Complete)
|
||||
assert.Equal(t, uint64(4), tree.Total)
|
||||
assert.Equal(t, []string{"root", "a", "a-child", "b"}, summaryIDs(tree.Spans))
|
||||
})
|
||||
|
||||
t.Run("WideLevelOverLimit_ShowsItsFirstSpansInTimeOrder", func(t *testing.T) {
|
||||
specs := []skeletonSpec{{id: "root", startMs: 0, durationMs: 1000}}
|
||||
for i := range 200 {
|
||||
spec := skeletonSpec{id: fmt.Sprintf("s-%03d", i), parent: "root", startMs: int64(i), durationMs: 1}
|
||||
// A late error and a late long span do not change which spans are shown.
|
||||
switch i {
|
||||
case 150:
|
||||
spec.hasError = true
|
||||
case 170:
|
||||
spec.durationMs = 900
|
||||
}
|
||||
specs = append(specs, spec)
|
||||
}
|
||||
|
||||
tree := buildSkeleton(specs...).Tree()
|
||||
|
||||
assert.False(t, tree.Complete)
|
||||
assert.Equal(t, uint64(201), tree.Total)
|
||||
ids := summaryIDs(tree.Spans)
|
||||
require.Len(t, ids, inspectTreeSpans)
|
||||
assert.Equal(t, "root", ids[0])
|
||||
assert.Equal(t, "s-000", ids[1])
|
||||
assert.Equal(t, "s-048", ids[49])
|
||||
assert.Equal(t, uint32(200), tree.Spans[0].ChildCount)
|
||||
})
|
||||
|
||||
t.Run("NarrowTopLevels_DescendsUntilLimit", func(t *testing.T) {
|
||||
specs := []skeletonSpec{
|
||||
{id: "root", startMs: 0, durationMs: 1000},
|
||||
{id: "gateway", parent: "root", startMs: 1, durationMs: 900},
|
||||
{id: "handler", parent: "gateway", startMs: 2, durationMs: 800},
|
||||
}
|
||||
for i := range 150 {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("q-%03d", i), parent: "handler", startMs: int64(3 + i), durationMs: 1})
|
||||
}
|
||||
|
||||
tree := buildSkeleton(specs...).tree(10)
|
||||
|
||||
assert.Equal(t, []string{"root", "gateway", "handler", "q-000", "q-001", "q-002", "q-003", "q-004", "q-005", "q-006"}, summaryIDs(tree.Spans))
|
||||
assert.Equal(t, uint32(150), tree.Spans[2].ChildCount)
|
||||
})
|
||||
|
||||
t.Run("LevelCutAcrossParents_KeepsTreeOrder", func(t *testing.T) {
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", startMs: 0, durationMs: 100},
|
||||
skeletonSpec{id: "a", parent: "root", startMs: 1, durationMs: 40},
|
||||
skeletonSpec{id: "b", parent: "root", startMs: 50, durationMs: 40},
|
||||
skeletonSpec{id: "a-1", parent: "a", startMs: 2, durationMs: 1},
|
||||
skeletonSpec{id: "a-2", parent: "a", startMs: 3, durationMs: 1},
|
||||
skeletonSpec{id: "b-1", parent: "b", startMs: 51, durationMs: 1},
|
||||
)
|
||||
|
||||
tree := skeleton.tree(5)
|
||||
|
||||
assert.False(t, tree.Complete)
|
||||
assert.Equal(t, []string{"root", "a", "a-1", "a-2", "b"}, summaryIDs(tree.Spans))
|
||||
})
|
||||
}
|
||||
|
||||
func TestFitOverview(t *testing.T) {
|
||||
// A root with 99 children that each fail with a 404; long names make every line large.
|
||||
traceWith := func(name string) *TraceSkeleton {
|
||||
specs := []skeletonSpec{{id: "root", name: "root", service: "checkout-service", startMs: 0, durationMs: 1000}}
|
||||
for i := range 99 {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("span-%03d", i), parent: "root", name: fmt.Sprintf("%s %03d", name, i), service: "checkout-service", startMs: int64(i), durationMs: 1, statusCode: "404"})
|
||||
}
|
||||
return buildSkeleton(specs...)
|
||||
}
|
||||
newOverview := func(skeleton *TraceSkeleton) *TraceOverview {
|
||||
overview := &TraceOverview{Identity: skeleton.Identity(), UnmarkedFailures: skeleton.UnmarkedFailurePage(0), Slowest: skeleton.SlowestPage(0), Tree: skeleton.Tree()}
|
||||
overview.Tree.Records = []SpanRecord{{SpanID: "root"}}
|
||||
return overview
|
||||
}
|
||||
|
||||
t.Run("OverviewOverSizeLimit_DropsRecordsThenShortensTheTree", func(t *testing.T) {
|
||||
skeleton := traceWith(strings.Repeat("operation ", 40))
|
||||
overview := newOverview(skeleton)
|
||||
require.Greater(t, jsonChars(overview), inspectMaxResponseChars)
|
||||
|
||||
skeleton.FitOverview(overview)
|
||||
|
||||
// The tree is cut to what fits and no further: two more spans would pass the limit.
|
||||
assert.LessOrEqual(t, jsonChars(overview), inspectMaxResponseChars)
|
||||
assert.Greater(t, jsonChars(overview), inspectMaxResponseChars-2*jsonChars(overview.Tree.Spans[1]))
|
||||
assert.Nil(t, overview.Tree.Records)
|
||||
assert.Less(t, len(overview.Tree.Spans), inspectTreeSpans)
|
||||
assert.False(t, overview.Tree.Complete)
|
||||
assert.Equal(t, uint64(100), overview.Tree.Total)
|
||||
assert.Equal(t, "root", overview.Tree.Spans[0].SpanID)
|
||||
assert.Len(t, overview.Slowest.Items, inspectListSize)
|
||||
})
|
||||
|
||||
t.Run("OverviewWithinSizeLimit_IsLeftAsItIs", func(t *testing.T) {
|
||||
skeleton := traceWith("op")
|
||||
overview := newOverview(skeleton)
|
||||
require.LessOrEqual(t, jsonChars(overview), inspectMaxResponseChars)
|
||||
|
||||
skeleton.FitOverview(overview)
|
||||
|
||||
assert.Len(t, overview.Tree.Spans, inspectTreeSpans)
|
||||
assert.NotNil(t, overview.Tree.Records)
|
||||
})
|
||||
}
|
||||
|
||||
func TestIdentity(t *testing.T) {
|
||||
inputTokens, outputTokens, cost := 1200.0, 300.0, 0.02
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", name: "POST /chat", service: "web", startMs: 0, durationMs: 100, statusCode: "200"},
|
||||
skeletonSpec{id: "agent", parent: "root", name: "invoke_agent", service: "agent", startMs: 10, durationMs: 80},
|
||||
skeletonSpec{id: "model", parent: "agent", name: "chat gpt", service: "agent", startMs: 20, durationMs: 50, hasError: true},
|
||||
skeletonSpec{id: "tool", parent: "agent", name: "execute_tool lookup", service: "agent", startMs: 75, durationMs: 10},
|
||||
).WithAISpans([]AISpanUsage{
|
||||
{SpanID: "model", IsModelCall: true, InputTokens: &inputTokens, OutputTokens: &outputTokens, Cost: &cost},
|
||||
{SpanID: "tool", IsToolCall: true},
|
||||
{SpanID: "agent"},
|
||||
})
|
||||
|
||||
identity := skeleton.Identity()
|
||||
|
||||
assert.Equal(t, uint64(100_000), identity.StartTimestampMillis)
|
||||
assert.Equal(t, uint64(100*time.Millisecond), identity.DurationNano)
|
||||
require.NotNil(t, identity.Root)
|
||||
assert.Equal(t, "POST /chat", identity.Root.Name)
|
||||
assert.Equal(t, uint64(4), identity.SpanCount)
|
||||
assert.Equal(t, uint64(1), identity.ErrorSpanCount)
|
||||
assert.Equal(t, uint64(2), identity.ServiceCount)
|
||||
assert.Equal(t, []TraceServiceStats{
|
||||
{ServiceName: "agent", SpanCount: 3, ErrorSpanCount: 1, OwnTimeNano: uint64(80 * time.Millisecond)},
|
||||
{ServiceName: "web", SpanCount: 1, ErrorSpanCount: 0, OwnTimeNano: uint64(20 * time.Millisecond)},
|
||||
}, identity.Services)
|
||||
require.NotNil(t, identity.AI)
|
||||
assert.Equal(t, uint64(1), identity.AI.ModelCallCount)
|
||||
assert.Equal(t, uint64(1), identity.AI.ToolCallCount)
|
||||
assert.Equal(t, uint64(1200), identity.AI.Tokens.Input)
|
||||
require.NotNil(t, identity.AI.TotalCost)
|
||||
assert.InDelta(t, 0.02, *identity.AI.TotalCost, 1e-9)
|
||||
|
||||
assert.Nil(t, buildSkeleton(skeletonSpec{id: "plain", startMs: 0, durationMs: 1}).Identity().AI)
|
||||
}
|
||||
|
||||
func TestExpensivePage(t *testing.T) {
|
||||
t.Run("WithCost_RanksByCost", func(t *testing.T) {
|
||||
cheap, costly, manyTokens := 0.01, 0.5, 9000.0
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", startMs: 0, durationMs: 100},
|
||||
skeletonSpec{id: "cheap", parent: "root", startMs: 1, durationMs: 10},
|
||||
skeletonSpec{id: "costly", parent: "root", startMs: 2, durationMs: 10},
|
||||
).WithAISpans([]AISpanUsage{
|
||||
{SpanID: "cheap", IsModelCall: true, InputTokens: &manyTokens, Cost: &cheap},
|
||||
{SpanID: "costly", IsModelCall: true, Cost: &costly},
|
||||
})
|
||||
|
||||
expensive := skeleton.ExpensivePage(0)
|
||||
|
||||
require.NotNil(t, expensive)
|
||||
assert.Equal(t, uint64(2), expensive.Total)
|
||||
assert.Equal(t, []string{"costly", "cheap"}, []string{expensive.Items[0].Span.SpanID, expensive.Items[1].Span.SpanID})
|
||||
require.NotNil(t, expensive.Items[0].Parent)
|
||||
assert.Equal(t, "root", expensive.Items[0].Parent.SpanID)
|
||||
assert.Equal(t, uint64(9000), expensive.Items[1].InputTokens)
|
||||
})
|
||||
|
||||
t.Run("WithoutCost_RanksByTokens", func(t *testing.T) {
|
||||
few, many := 10.0, 9000.0
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "small", startMs: 0, durationMs: 10},
|
||||
skeletonSpec{id: "large", startMs: 1, durationMs: 10},
|
||||
).WithAISpans([]AISpanUsage{
|
||||
{SpanID: "small", IsModelCall: true, InputTokens: &few},
|
||||
{SpanID: "large", IsModelCall: true, InputTokens: &many},
|
||||
{SpanID: "not-loaded", IsModelCall: true, InputTokens: &many},
|
||||
})
|
||||
|
||||
expensive := skeleton.ExpensivePage(0)
|
||||
|
||||
assert.Equal(t, uint64(2), expensive.Total)
|
||||
assert.Equal(t, []string{"large", "small"}, []string{expensive.Items[0].Span.SpanID, expensive.Items[1].Span.SpanID})
|
||||
assert.Nil(t, expensive.Items[0].Cost)
|
||||
})
|
||||
|
||||
t.Run("MoreThanOnePage_PagesEqualCostsInSpanOrder", func(t *testing.T) {
|
||||
cost := 0.25
|
||||
specs := []skeletonSpec{{id: "root", startMs: 0, durationMs: 1000}}
|
||||
usage := make([]AISpanUsage, 0, 14)
|
||||
for i := range 14 {
|
||||
id := fmt.Sprintf("call-%02d", i)
|
||||
specs = append(specs, skeletonSpec{id: id, parent: "root", startMs: int64(i + 1), durationMs: 1})
|
||||
// Given in reverse so the page order has to come from the spans, not from the input.
|
||||
usage = append([]AISpanUsage{{SpanID: id, IsModelCall: true, Cost: &cost}}, usage...)
|
||||
}
|
||||
skeleton := buildSkeleton(specs...).WithAISpans(usage)
|
||||
|
||||
first := skeleton.ExpensivePage(0)
|
||||
second := skeleton.ExpensivePage(nextOffset(t, OverviewListExpensive, first.NextCursor))
|
||||
|
||||
assert.Equal(t, uint64(14), first.Total)
|
||||
assert.Equal(t, "call-00", first.Items[0].Span.SpanID)
|
||||
assert.Equal(t, "call-09", first.Items[9].Span.SpanID)
|
||||
require.Len(t, second.Items, 4)
|
||||
assert.Equal(t, "call-13", second.Items[3].Span.SpanID)
|
||||
assert.Empty(t, second.NextCursor)
|
||||
})
|
||||
|
||||
t.Run("NoGenAISpans_ReturnsNothing", func(t *testing.T) {
|
||||
assert.Nil(t, buildSkeleton(skeletonSpec{id: "plain", startMs: 0, durationMs: 1}).ExpensivePage(0))
|
||||
})
|
||||
}
|
||||
454
pkg/types/spantypes/trace_skeleton.go
Normal file
454
pkg/types/spantypes/trace_skeleton.go
Normal file
@@ -0,0 +1,454 @@
|
||||
package spantypes
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
skeletonFlagError uint8 = 1 << iota
|
||||
skeletonFlagExceptionEvent
|
||||
skeletonFlagMissingParent
|
||||
)
|
||||
|
||||
const noParent int32 = -1
|
||||
|
||||
// SkeletonSpan is the ClickHouse scan struct for the columns a TraceSkeleton is built from.
|
||||
type SkeletonSpan struct {
|
||||
SpanID string `ch:"span_id"`
|
||||
ParentSpanID string `ch:"parent_span_id"`
|
||||
StartTime time.Time `ch:"timestamp"`
|
||||
DurationNano uint64 `ch:"duration_nano"`
|
||||
HasError bool `ch:"has_error"`
|
||||
Name string `ch:"name"`
|
||||
ServiceName string `ch:"resource_string_service$$name"`
|
||||
KindString string `ch:"kind_string"`
|
||||
ResponseStatusCode string `ch:"response_status_code"`
|
||||
HasExceptionEvent bool `ch:"has_exception_event"`
|
||||
}
|
||||
|
||||
// AISpanUsage is the token and cost usage of one gen_ai span.
|
||||
type AISpanUsage struct {
|
||||
SpanID string `ch:"span_id"`
|
||||
IsModelCall bool `ch:"is_model_call"`
|
||||
IsToolCall bool `ch:"is_tool_call"`
|
||||
InputTokens *float64 `ch:"input_tokens_value"`
|
||||
OutputTokens *float64 `ch:"output_tokens_value"`
|
||||
CacheReadTokens *float64 `ch:"cache_read_tokens_value"`
|
||||
CacheWriteTokens *float64 `ch:"cache_write_tokens_value"`
|
||||
ReasoningTokens *float64 `ch:"reasoning_tokens_value"`
|
||||
Cost *float64 `ch:"total_cost_value"`
|
||||
}
|
||||
|
||||
// LinkedSpan is a span that links to other spans, in its own trace or another one.
|
||||
type LinkedSpan struct {
|
||||
SpanID string
|
||||
Links []Link
|
||||
}
|
||||
|
||||
// NewLinkedSpan reads the links out of a span's stored references; ok is false when it has none.
|
||||
func NewLinkedSpan(spanID, references string) (LinkedSpan, bool) {
|
||||
refs := []OtelSpanRef{}
|
||||
if err := json.Unmarshal([]byte(references), &refs); err != nil {
|
||||
return LinkedSpan{}, false
|
||||
}
|
||||
linked := LinkedSpan{SpanID: spanID, Links: make([]Link, 0, len(refs))}
|
||||
for _, ref := range refs {
|
||||
// The parent is stored as a CHILD_OF reference beside the links.
|
||||
if ref.RefType != "CHILD_OF" {
|
||||
linked.Links = append(linked.Links, Link{TraceID: ref.TraceId, SpanID: ref.SpanId})
|
||||
}
|
||||
}
|
||||
return linked, len(linked.Links) > 0
|
||||
}
|
||||
|
||||
// TraceSkeleton holds the tree of a trace as parallel arrays indexed by span position, in start
|
||||
// time order. It is read-only once built.
|
||||
type TraceSkeleton struct {
|
||||
TraceID string
|
||||
|
||||
// SpanIDs holds every span id back to back; span i is SpanIDs[IDOffsets[i]:IDOffsets[i+1]].
|
||||
SpanIDs string
|
||||
IDOffsets []uint32
|
||||
// Parents holds the position of each span's parent, or -1 for a root.
|
||||
Parents []int32
|
||||
StartNano []int64
|
||||
DurationNano []uint64
|
||||
Flags []uint8
|
||||
// Names, Services, Kinds and StatusCodes index into Strings.
|
||||
Names []uint32
|
||||
Services []uint32
|
||||
Kinds []uint32
|
||||
StatusCodes []uint32
|
||||
Strings []string
|
||||
|
||||
AISpans []AISpanUsage
|
||||
// LinkedSpans holds the spans with links, in span order.
|
||||
LinkedSpans []LinkedSpan
|
||||
|
||||
// Children of span i are childList[childStart[i]:childStart[i+1]], in span order.
|
||||
childStart []int32
|
||||
childList []int32
|
||||
roots []int32
|
||||
// preorder lists span positions depth first, roots and siblings in span order.
|
||||
preorder []int32
|
||||
level []uint32
|
||||
// subtree counts the spans below each span, the span itself excluded.
|
||||
subtree []uint32
|
||||
ownTime []uint64
|
||||
// idOrder lists span positions sorted by span id, for lookups by id.
|
||||
idOrder []int32
|
||||
startNs int64
|
||||
endNs int64
|
||||
errCount uint64
|
||||
}
|
||||
|
||||
// TraceSkeletonBuilder collects spans in (start time, name, span id) order.
|
||||
type TraceSkeletonBuilder struct {
|
||||
skeleton *TraceSkeleton
|
||||
ids []byte
|
||||
parentIDs []string
|
||||
interned map[string]uint32
|
||||
}
|
||||
|
||||
func NewTraceSkeletonBuilder(traceID string, sizeHint int) *TraceSkeletonBuilder {
|
||||
return &TraceSkeletonBuilder{
|
||||
skeleton: &TraceSkeleton{
|
||||
TraceID: traceID,
|
||||
IDOffsets: make([]uint32, 1, sizeHint+1),
|
||||
Parents: make([]int32, 0, sizeHint),
|
||||
StartNano: make([]int64, 0, sizeHint),
|
||||
DurationNano: make([]uint64, 0, sizeHint),
|
||||
Flags: make([]uint8, 0, sizeHint),
|
||||
Names: make([]uint32, 0, sizeHint),
|
||||
Services: make([]uint32, 0, sizeHint),
|
||||
Kinds: make([]uint32, 0, sizeHint),
|
||||
StatusCodes: make([]uint32, 0, sizeHint),
|
||||
AISpans: []AISpanUsage{},
|
||||
LinkedSpans: []LinkedSpan{},
|
||||
},
|
||||
ids: make([]byte, 0, sizeHint*16),
|
||||
parentIDs: make([]string, 0, sizeHint),
|
||||
interned: make(map[string]uint32),
|
||||
}
|
||||
}
|
||||
|
||||
func (b *TraceSkeletonBuilder) Add(span *SkeletonSpan) {
|
||||
s := b.skeleton
|
||||
b.ids = append(b.ids, span.SpanID...)
|
||||
s.IDOffsets = append(s.IDOffsets, uint32(len(b.ids)))
|
||||
b.parentIDs = append(b.parentIDs, span.ParentSpanID)
|
||||
s.StartNano = append(s.StartNano, span.StartTime.UnixNano())
|
||||
s.DurationNano = append(s.DurationNano, span.DurationNano)
|
||||
var flags uint8
|
||||
if span.HasError {
|
||||
flags |= skeletonFlagError
|
||||
}
|
||||
if span.HasExceptionEvent {
|
||||
flags |= skeletonFlagExceptionEvent
|
||||
}
|
||||
s.Flags = append(s.Flags, flags)
|
||||
s.Names = append(s.Names, b.intern(span.Name))
|
||||
s.Services = append(s.Services, b.intern(span.ServiceName))
|
||||
s.Kinds = append(s.Kinds, b.intern(span.KindString))
|
||||
s.StatusCodes = append(s.StatusCodes, b.intern(span.ResponseStatusCode))
|
||||
}
|
||||
|
||||
func (b *TraceSkeletonBuilder) Len() int {
|
||||
return len(b.parentIDs)
|
||||
}
|
||||
|
||||
// Build links each span to its parent and computes the tree fields. The builder cannot be reused.
|
||||
func (b *TraceSkeletonBuilder) Build() *TraceSkeleton {
|
||||
s := b.skeleton
|
||||
s.SpanIDs = string(b.ids)
|
||||
s.buildIDOrder()
|
||||
|
||||
n := len(b.parentIDs)
|
||||
s.Parents = s.Parents[:0]
|
||||
for i := range n {
|
||||
parent := noParent
|
||||
if parentID := b.parentIDs[i]; parentID != "" {
|
||||
if idx, ok := s.Index(parentID); ok && idx != int32(i) {
|
||||
parent = idx
|
||||
} else {
|
||||
s.Flags[i] |= skeletonFlagMissingParent
|
||||
}
|
||||
}
|
||||
s.Parents = append(s.Parents, parent)
|
||||
}
|
||||
b.parentIDs, b.interned, b.ids = nil, nil, nil
|
||||
s.derive()
|
||||
return s
|
||||
}
|
||||
|
||||
func (b *TraceSkeletonBuilder) intern(value string) uint32 {
|
||||
if idx, ok := b.interned[value]; ok {
|
||||
return idx
|
||||
}
|
||||
idx := uint32(len(b.skeleton.Strings))
|
||||
b.skeleton.Strings = append(b.skeleton.Strings, value)
|
||||
b.interned[value] = idx
|
||||
return idx
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) Len() int {
|
||||
return len(s.Parents)
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) SpanID(i int32) string {
|
||||
return s.SpanIDs[s.IDOffsets[i]:s.IDOffsets[i+1]]
|
||||
}
|
||||
|
||||
// Index returns the position of the span with the given id.
|
||||
func (s *TraceSkeleton) Index(spanID string) (int32, bool) {
|
||||
pos, found := sort.Find(len(s.idOrder), func(k int) int {
|
||||
id := s.SpanID(s.idOrder[k])
|
||||
switch {
|
||||
case spanID < id:
|
||||
return -1
|
||||
case spanID > id:
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
if !found {
|
||||
return 0, false
|
||||
}
|
||||
return s.idOrder[pos], true
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) Children(i int32) []int32 {
|
||||
return s.childList[s.childStart[i]:s.childStart[i+1]]
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) HasError(i int32) bool {
|
||||
return s.Flags[i]&skeletonFlagError != 0
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) HasExceptionEvent(i int32) bool {
|
||||
return s.Flags[i]&skeletonFlagExceptionEvent != 0
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) HasMissingParent(i int32) bool {
|
||||
return s.Flags[i]&skeletonFlagMissingParent != 0
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) ResponseStatusCode(i int32) string {
|
||||
return s.Strings[s.StatusCodes[i]]
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) HasMissingSpans() bool {
|
||||
for _, root := range s.roots {
|
||||
if s.HasMissingParent(root) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// PrimaryRoot is the earliest span without a parent, or the earliest orphan when every root's
|
||||
// parent is missing.
|
||||
func (s *TraceSkeleton) PrimaryRoot() (int32, bool) {
|
||||
if len(s.roots) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
for _, root := range s.roots {
|
||||
if !s.HasMissingParent(root) {
|
||||
return root, true
|
||||
}
|
||||
}
|
||||
return s.roots[0], true
|
||||
}
|
||||
|
||||
// Summary returns the short form of span i.
|
||||
func (s *TraceSkeleton) Summary(i int32) SpanSummary {
|
||||
summary := SpanSummary{
|
||||
SpanID: s.SpanID(i),
|
||||
Name: s.Strings[s.Names[i]],
|
||||
ServiceName: s.Strings[s.Services[i]],
|
||||
Kind: s.Strings[s.Kinds[i]],
|
||||
StartOffsetNano: uint64(s.StartNano[i] - s.startNs),
|
||||
DurationNano: s.DurationNano[i],
|
||||
OwnTimeNano: s.ownTime[i],
|
||||
ResponseStatusCode: s.ResponseStatusCode(i),
|
||||
HasError: s.HasError(i),
|
||||
Level: s.level[i],
|
||||
ChildCount: uint32(s.childStart[i+1] - s.childStart[i]),
|
||||
SubtreeSpanCount: s.subtree[i],
|
||||
}
|
||||
if parent := s.Parents[i]; parent != noParent {
|
||||
summary.ParentSpanID = s.SpanID(parent)
|
||||
}
|
||||
return summary
|
||||
}
|
||||
|
||||
// WithAISpans returns a copy of the skeleton carrying the usage of its gen_ai spans.
|
||||
func (s *TraceSkeleton) WithAISpans(aiSpans []AISpanUsage) *TraceSkeleton {
|
||||
cp := *s
|
||||
cp.AISpans = aiSpans
|
||||
if cp.AISpans == nil {
|
||||
cp.AISpans = []AISpanUsage{}
|
||||
}
|
||||
return &cp
|
||||
}
|
||||
|
||||
// WithLinkedSpans returns a copy of the skeleton carrying its spans that have links, in span order.
|
||||
func (s *TraceSkeleton) WithLinkedSpans(linkedSpans []LinkedSpan) *TraceSkeleton {
|
||||
type positioned struct {
|
||||
index int32
|
||||
linked LinkedSpan
|
||||
}
|
||||
known := make([]positioned, 0, len(linkedSpans))
|
||||
for _, linked := range linkedSpans {
|
||||
if idx, ok := s.Index(linked.SpanID); ok {
|
||||
known = append(known, positioned{index: idx, linked: linked})
|
||||
}
|
||||
}
|
||||
slices.SortFunc(known, func(a, b positioned) int { return int(a.index) - int(b.index) })
|
||||
|
||||
cp := *s
|
||||
cp.LinkedSpans = make([]LinkedSpan, 0, len(known))
|
||||
for _, item := range known {
|
||||
cp.LinkedSpans = append(cp.LinkedSpans, item.linked)
|
||||
}
|
||||
return &cp
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) buildIDOrder() {
|
||||
n := len(s.IDOffsets) - 1
|
||||
s.idOrder = make([]int32, n)
|
||||
for i := range s.idOrder {
|
||||
s.idOrder[i] = int32(i)
|
||||
}
|
||||
slices.SortFunc(s.idOrder, func(a, b int32) int {
|
||||
idA, idB := s.SpanID(a), s.SpanID(b)
|
||||
switch {
|
||||
case idA < idB:
|
||||
return -1
|
||||
case idA > idB:
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) derive() {
|
||||
n := s.Len()
|
||||
s.startNs, s.endNs, s.errCount = 0, 0, 0
|
||||
for i := range n {
|
||||
if i == 0 || s.StartNano[i] < s.startNs {
|
||||
s.startNs = s.StartNano[i]
|
||||
}
|
||||
s.endNs = max(s.endNs, s.StartNano[i]+int64(s.DurationNano[i]))
|
||||
if s.Flags[i]&skeletonFlagError != 0 {
|
||||
s.errCount++
|
||||
}
|
||||
}
|
||||
|
||||
s.buildChildren()
|
||||
// A span left unvisited sits on a parent cycle; cutting its parent link makes it a root.
|
||||
if s.walk() < n {
|
||||
visited := make([]bool, n)
|
||||
for _, i := range s.preorder {
|
||||
visited[i] = true
|
||||
}
|
||||
for i := range n {
|
||||
if visited[i] {
|
||||
continue
|
||||
}
|
||||
s.Parents[i] = noParent
|
||||
s.Flags[i] |= skeletonFlagMissingParent
|
||||
stack := []int32{int32(i)}
|
||||
for len(stack) > 0 {
|
||||
top := stack[len(stack)-1]
|
||||
stack = stack[:len(stack)-1]
|
||||
if visited[top] {
|
||||
continue
|
||||
}
|
||||
visited[top] = true
|
||||
stack = append(stack, s.Children(top)...)
|
||||
}
|
||||
}
|
||||
s.buildChildren()
|
||||
s.walk()
|
||||
}
|
||||
|
||||
s.subtree = make([]uint32, n)
|
||||
for k := n - 1; k >= 0; k-- {
|
||||
i := s.preorder[k]
|
||||
if parent := s.Parents[i]; parent != noParent {
|
||||
s.subtree[parent] += s.subtree[i] + 1
|
||||
}
|
||||
}
|
||||
|
||||
s.ownTime = make([]uint64, n)
|
||||
for i := range n {
|
||||
s.ownTime[i] = s.DurationNano[i] - s.childrenCoverNano(int32(i))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) buildChildren() {
|
||||
n := s.Len()
|
||||
s.childStart = make([]int32, n+1)
|
||||
s.roots = s.roots[:0]
|
||||
for i := range n {
|
||||
if parent := s.Parents[i]; parent != noParent {
|
||||
s.childStart[parent+1]++
|
||||
} else {
|
||||
s.roots = append(s.roots, int32(i))
|
||||
}
|
||||
}
|
||||
for i := range n {
|
||||
s.childStart[i+1] += s.childStart[i]
|
||||
}
|
||||
s.childList = make([]int32, s.childStart[n])
|
||||
next := slices.Clone(s.childStart[:n])
|
||||
for i := range n {
|
||||
if parent := s.Parents[i]; parent != noParent {
|
||||
s.childList[next[parent]] = int32(i)
|
||||
next[parent]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// walk fills preorder and level from the roots and returns the number of spans reached.
|
||||
func (s *TraceSkeleton) walk() int {
|
||||
n := s.Len()
|
||||
s.preorder = make([]int32, 0, n)
|
||||
s.level = make([]uint32, n)
|
||||
stack := make([]int32, 0, 64)
|
||||
for r := len(s.roots) - 1; r >= 0; r-- {
|
||||
stack = append(stack, s.roots[r])
|
||||
}
|
||||
for len(stack) > 0 {
|
||||
i := stack[len(stack)-1]
|
||||
stack = stack[:len(stack)-1]
|
||||
s.preorder = append(s.preorder, i)
|
||||
children := s.Children(i)
|
||||
for c := len(children) - 1; c >= 0; c-- {
|
||||
s.level[children[c]] = s.level[i] + 1
|
||||
stack = append(stack, children[c])
|
||||
}
|
||||
}
|
||||
return len(s.preorder)
|
||||
}
|
||||
|
||||
// childrenCoverNano is the time of span i its children cover, overlapping children counted once.
|
||||
func (s *TraceSkeleton) childrenCoverNano(i int32) uint64 {
|
||||
start := s.StartNano[i]
|
||||
end := start + int64(s.DurationNano[i])
|
||||
covered, cursor := int64(0), start
|
||||
for _, child := range s.Children(i) {
|
||||
childStart := max(s.StartNano[child], cursor)
|
||||
childEnd := min(s.StartNano[child]+int64(s.DurationNano[child]), end)
|
||||
if childEnd <= childStart {
|
||||
continue
|
||||
}
|
||||
covered += childEnd - childStart
|
||||
cursor = childEnd
|
||||
}
|
||||
return uint64(covered)
|
||||
}
|
||||
217
pkg/types/spantypes/trace_skeleton_test.go
Normal file
217
pkg/types/spantypes/trace_skeleton_test.go
Normal file
@@ -0,0 +1,217 @@
|
||||
package spantypes
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type skeletonSpec struct {
|
||||
id, parent, name, service string
|
||||
startMs, durationMs int64
|
||||
hasError, hasException bool
|
||||
statusCode string
|
||||
}
|
||||
|
||||
// buildSkeleton adds the spans in the given order, which stands for the store's start time order.
|
||||
func buildSkeleton(specs ...skeletonSpec) *TraceSkeleton {
|
||||
builder := NewTraceSkeletonBuilder("trace", len(specs))
|
||||
for _, spec := range specs {
|
||||
builder.Add(&SkeletonSpan{
|
||||
SpanID: spec.id,
|
||||
ParentSpanID: spec.parent,
|
||||
StartTime: time.Unix(100, 0).Add(time.Duration(spec.startMs) * time.Millisecond),
|
||||
DurationNano: uint64(spec.durationMs) * uint64(time.Millisecond),
|
||||
HasError: spec.hasError,
|
||||
Name: spec.name,
|
||||
ServiceName: spec.service,
|
||||
KindString: "Internal",
|
||||
ResponseStatusCode: spec.statusCode,
|
||||
HasExceptionEvent: spec.hasException,
|
||||
})
|
||||
}
|
||||
return builder.Build()
|
||||
}
|
||||
|
||||
func ownTimeMs(t *testing.T, skeleton *TraceSkeleton, spanID string) int64 {
|
||||
t.Helper()
|
||||
idx, ok := skeleton.Index(spanID)
|
||||
require.True(t, ok, "span %s", spanID)
|
||||
return int64(skeleton.ownTime[idx] / uint64(time.Millisecond))
|
||||
}
|
||||
|
||||
func TestTraceSkeletonOwnTime(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
specs []skeletonSpec
|
||||
expected int64
|
||||
}{
|
||||
{
|
||||
name: "SequentialChildren_SubtractsEach",
|
||||
specs: []skeletonSpec{
|
||||
{id: "root", startMs: 0, durationMs: 100},
|
||||
{id: "a", parent: "root", startMs: 10, durationMs: 20},
|
||||
{id: "b", parent: "root", startMs: 50, durationMs: 30},
|
||||
},
|
||||
expected: 50,
|
||||
},
|
||||
{
|
||||
name: "ParallelChildren_CountsOverlapOnce",
|
||||
specs: []skeletonSpec{
|
||||
{id: "root", startMs: 0, durationMs: 100},
|
||||
{id: "a", parent: "root", startMs: 10, durationMs: 40},
|
||||
{id: "b", parent: "root", startMs: 20, durationMs: 40},
|
||||
{id: "c", parent: "root", startMs: 30, durationMs: 10},
|
||||
},
|
||||
expected: 50,
|
||||
},
|
||||
{
|
||||
name: "ChildOutlivesParent_ClipsToParentEnd",
|
||||
specs: []skeletonSpec{
|
||||
{id: "root", startMs: 0, durationMs: 100},
|
||||
{id: "a", parent: "root", startMs: 80, durationMs: 500},
|
||||
},
|
||||
expected: 80,
|
||||
},
|
||||
{
|
||||
name: "ChildStartsBeforeParent_ClipsToParentStart",
|
||||
specs: []skeletonSpec{
|
||||
{id: "early", parent: "root", startMs: 0, durationMs: 30},
|
||||
{id: "root", startMs: 10, durationMs: 100},
|
||||
},
|
||||
expected: 80,
|
||||
},
|
||||
{
|
||||
name: "ChildCoversParent_LeavesZero",
|
||||
specs: []skeletonSpec{
|
||||
{id: "early", parent: "root", startMs: 0, durationMs: 500},
|
||||
{id: "root", startMs: 10, durationMs: 100},
|
||||
},
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "NoChildren_EqualsDuration",
|
||||
specs: []skeletonSpec{
|
||||
{id: "root", startMs: 0, durationMs: 70},
|
||||
},
|
||||
expected: 70,
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
skeleton := buildSkeleton(testCase.specs...)
|
||||
assert.Equal(t, testCase.expected, ownTimeMs(t, skeleton, "root"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTraceSkeletonTree(t *testing.T) {
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", name: "GET /", service: "web", startMs: 0, durationMs: 100},
|
||||
skeletonSpec{id: "a", parent: "root", name: "a", service: "web", startMs: 10, durationMs: 30},
|
||||
skeletonSpec{id: "a1", parent: "a", name: "a1", service: "db", startMs: 15, durationMs: 5, hasError: true},
|
||||
skeletonSpec{id: "b", parent: "root", name: "b", service: "web", startMs: 50, durationMs: 10},
|
||||
skeletonSpec{id: "orphan", parent: "gone", name: "orphan", service: "worker", startMs: 60, durationMs: 10},
|
||||
)
|
||||
|
||||
order := make([]string, 0, skeleton.Len())
|
||||
for _, i := range skeleton.preorder {
|
||||
order = append(order, skeleton.SpanID(i))
|
||||
}
|
||||
assert.Equal(t, []string{"root", "a", "a1", "b", "orphan"}, order)
|
||||
assert.Len(t, skeleton.roots, 2)
|
||||
assert.True(t, skeleton.HasMissingSpans())
|
||||
assert.Equal(t, uint64(1), skeleton.errCount)
|
||||
|
||||
root, ok := skeleton.PrimaryRoot()
|
||||
require.True(t, ok)
|
||||
rootSummary := skeleton.Summary(root)
|
||||
assert.Equal(t, "root", rootSummary.SpanID)
|
||||
assert.Equal(t, uint32(2), rootSummary.ChildCount)
|
||||
assert.Equal(t, uint32(3), rootSummary.SubtreeSpanCount)
|
||||
|
||||
leaf, ok := skeleton.Index("a1")
|
||||
require.True(t, ok)
|
||||
leafSummary := skeleton.Summary(leaf)
|
||||
assert.Equal(t, uint32(2), leafSummary.Level)
|
||||
assert.Equal(t, "a", leafSummary.ParentSpanID)
|
||||
assert.Equal(t, uint64(15*time.Millisecond), leafSummary.StartOffsetNano)
|
||||
assert.True(t, leafSummary.HasError)
|
||||
assert.Equal(t, "db", leafSummary.ServiceName)
|
||||
|
||||
_, ok = skeleton.Index("gone")
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestTraceSkeletonPrimaryRoot(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
specs []skeletonSpec
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "OrphanStartsFirst_PrefersSpanWithoutParent",
|
||||
specs: []skeletonSpec{
|
||||
{id: "orphan", parent: "gone", startMs: 0, durationMs: 10},
|
||||
{id: "root", startMs: 5, durationMs: 100},
|
||||
},
|
||||
expected: "root",
|
||||
},
|
||||
{
|
||||
name: "OnlyOrphans_TakesEarliest",
|
||||
specs: []skeletonSpec{
|
||||
{id: "first", parent: "gone", startMs: 0, durationMs: 10},
|
||||
{id: "second", parent: "missing", startMs: 5, durationMs: 100},
|
||||
},
|
||||
expected: "first",
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
skeleton := buildSkeleton(testCase.specs...)
|
||||
root, ok := skeleton.PrimaryRoot()
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, testCase.expected, skeleton.SpanID(root))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTraceSkeletonParentCycle(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
specs []skeletonSpec
|
||||
}{
|
||||
{
|
||||
name: "TwoSpansPointAtEachOther",
|
||||
specs: []skeletonSpec{
|
||||
{id: "root", startMs: 0, durationMs: 10},
|
||||
{id: "x", parent: "y", startMs: 1, durationMs: 5},
|
||||
{id: "y", parent: "x", startMs: 2, durationMs: 5},
|
||||
{id: "under-y", parent: "y", startMs: 3, durationMs: 1},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "SpanIsItsOwnParent",
|
||||
specs: []skeletonSpec{
|
||||
{id: "self", parent: "self", startMs: 0, durationMs: 10},
|
||||
{id: "child", parent: "self", startMs: 1, durationMs: 5},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
skeleton := buildSkeleton(testCase.specs...)
|
||||
assert.Len(t, skeleton.preorder, len(testCase.specs))
|
||||
total := 0
|
||||
for _, root := range skeleton.roots {
|
||||
total += int(skeleton.Summary(root).SubtreeSpanCount) + 1
|
||||
}
|
||||
assert.Equal(t, len(testCase.specs), total)
|
||||
})
|
||||
}
|
||||
}
|
||||
153
pkg/types/spantypes/trace_span_record.go
Normal file
153
pkg/types/spantypes/trace_span_record.go
Normal file
@@ -0,0 +1,153 @@
|
||||
package spantypes
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
spanRecordValueChars = 500
|
||||
spanRecordChars = 20_000
|
||||
spanRecordShrunkChars = 100
|
||||
spanRecordMinimumChars = 20
|
||||
truncationMark = "…"
|
||||
)
|
||||
|
||||
// TruncatedField names a value of a span record that was cut, with its full length in characters.
|
||||
type TruncatedField struct {
|
||||
Field string `json:"field" required:"true"`
|
||||
Length uint64 `json:"length" required:"true"`
|
||||
}
|
||||
|
||||
// SpanRecord is everything stored on a span. Long values are cut and listed in Truncated.
|
||||
type SpanRecord struct {
|
||||
SpanID string `json:"spanId" required:"true"`
|
||||
StatusCode string `json:"statusCode" required:"true"`
|
||||
StatusMessage string `json:"statusMessage" required:"true"`
|
||||
Attributes map[string]any `json:"attributes" required:"true" nullable:"false"`
|
||||
Resource map[string]string `json:"resource" required:"true" nullable:"false"`
|
||||
Events []EventV2 `json:"events" required:"true" nullable:"false"`
|
||||
// Links holds the spans this span links to, in this or another trace.
|
||||
Links []Link `json:"links" required:"true" nullable:"false"`
|
||||
Truncated []TruncatedField `json:"truncated" required:"true" nullable:"false"`
|
||||
}
|
||||
|
||||
// NewSpanRecord cuts values over 500 characters, and shorter still while the record is over its size limit.
|
||||
func NewSpanRecord(item *StorableSpan) SpanRecord {
|
||||
record := SpanRecord{
|
||||
SpanID: item.SpanID,
|
||||
StatusCode: item.StatusCodeString,
|
||||
StatusMessage: item.StatusMessage,
|
||||
Attributes: item.Attributes(),
|
||||
Resource: make(map[string]string, len(item.ResourcesString)),
|
||||
Events: make([]EventV2, 0, len(item.Events)),
|
||||
Links: make([]Link, 0),
|
||||
Truncated: []TruncatedField{},
|
||||
}
|
||||
for key, value := range item.ResourcesString {
|
||||
record.Resource[key] = value
|
||||
}
|
||||
for _, event := range item.UnmarshalledEvents() {
|
||||
record.Events = append(record.Events, EventV2{Name: event.Name, TimeUnixNano: event.TimeUnixNano, Attributes: event.AttributeMap, IsError: event.IsError})
|
||||
}
|
||||
for _, ref := range item.UnmarshalledRefs() {
|
||||
if ref.RefType != "CHILD_OF" {
|
||||
record.Links = append(record.Links, Link{TraceID: ref.TraceId, SpanID: ref.SpanId})
|
||||
}
|
||||
}
|
||||
|
||||
record.cut(spanRecordValueChars)
|
||||
for _, limit := range []int{spanRecordShrunkChars, spanRecordMinimumChars} {
|
||||
if jsonChars(record) <= spanRecordChars {
|
||||
break
|
||||
}
|
||||
record.cut(limit)
|
||||
}
|
||||
slices.SortFunc(record.Truncated, func(a, b TruncatedField) int { return strings.Compare(a.Field, b.Field) })
|
||||
return record
|
||||
}
|
||||
|
||||
// cut shortens every string value longer than limit and records its full length once.
|
||||
func (r *SpanRecord) cut(limit int) {
|
||||
noted := make(map[string]struct{}, len(r.Truncated))
|
||||
for _, field := range r.Truncated {
|
||||
noted[field.Field] = struct{}{}
|
||||
}
|
||||
note := func(field string, length int) {
|
||||
if _, ok := noted[field]; !ok {
|
||||
noted[field] = struct{}{}
|
||||
r.Truncated = append(r.Truncated, TruncatedField{Field: field, Length: uint64(length)})
|
||||
}
|
||||
}
|
||||
|
||||
if cutValue, length, ok := cutString(r.StatusMessage, limit); ok {
|
||||
r.StatusMessage = cutValue
|
||||
note("statusMessage", length)
|
||||
}
|
||||
for key, value := range r.Attributes {
|
||||
if str, isString := value.(string); isString {
|
||||
if cutValue, length, ok := cutString(str, limit); ok {
|
||||
r.Attributes[key] = cutValue
|
||||
note("attributes."+key, length)
|
||||
}
|
||||
}
|
||||
}
|
||||
for key, value := range r.Resource {
|
||||
if cutValue, length, ok := cutString(value, limit); ok {
|
||||
r.Resource[key] = cutValue
|
||||
note("resource."+key, length)
|
||||
}
|
||||
}
|
||||
for idx := range r.Events {
|
||||
for key, value := range r.Events[idx].Attributes {
|
||||
if str, isString := value.(string); isString {
|
||||
if cutValue, length, ok := cutString(str, limit); ok {
|
||||
r.Events[idx].Attributes[key] = cutValue
|
||||
note(fmt.Sprintf("events[%d].attributes.%s", idx, key), length)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cutString returns value shortened to limit characters, and its length when it was longer or already cut.
|
||||
func cutString(value string, limit int) (string, int, bool) {
|
||||
if len(value) <= limit {
|
||||
return value, 0, false
|
||||
}
|
||||
length := utf8.RuneCountInString(value)
|
||||
if length <= limit {
|
||||
return value, 0, false
|
||||
}
|
||||
cut := 0
|
||||
for idx := range value {
|
||||
if cut == limit {
|
||||
return value[:idx] + truncationMark, length, true
|
||||
}
|
||||
cut++
|
||||
}
|
||||
return value, 0, false
|
||||
}
|
||||
|
||||
// SmallTraceRecords returns the full record of every span when the trace is small enough for all
|
||||
// of them to fit in the response; ok is false otherwise.
|
||||
func SmallTraceRecords(spans []StorableSpan) ([]SpanRecord, bool) {
|
||||
records := make([]SpanRecord, 0, len(spans))
|
||||
chars := 0
|
||||
for i := range spans {
|
||||
record := NewSpanRecord(&spans[i])
|
||||
chars += jsonChars(record)
|
||||
if chars > inspectSmallTraceChars {
|
||||
return nil, false
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
return records, true
|
||||
}
|
||||
|
||||
// IsSmallTrace reports whether a trace has few enough spans to try returning their records.
|
||||
func IsSmallTrace(spanCount int) bool {
|
||||
return spanCount <= inspectTreeSpans
|
||||
}
|
||||
84
pkg/types/spantypes/trace_span_record_test.go
Normal file
84
pkg/types/spantypes/trace_span_record_test.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package spantypes
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func recordSpan(spanID string, attributes map[string]string) StorableSpan {
|
||||
return StorableSpan{
|
||||
SpanID: spanID,
|
||||
Name: "POST /charge",
|
||||
ServiceName: "payments",
|
||||
SpanKind: "Server",
|
||||
StatusCodeString: "Error",
|
||||
StatusMessage: "worldpay declined the card",
|
||||
AttributesString: attributes,
|
||||
AttributesNumber: map[string]float64{"http.response.status_code": 502},
|
||||
ResourcesString: map[string]string{"service.name": "payments"},
|
||||
Events: []string{`{"name":"exception","timeUnixNano":5,"attributeMap":{"exception.message":"card declined"},"isError":true}`},
|
||||
References: `[{"traceId":"trace","spanId":"parent","refType":"CHILD_OF"},{"traceId":"other-trace","spanId":"producer","refType":"FOLLOWS_FROM"}]`,
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewSpanRecord(t *testing.T) {
|
||||
longStatement := strings.Repeat("SELECT 1; ", 120)
|
||||
|
||||
t.Run("LongValue_IsCutAndListed", func(t *testing.T) {
|
||||
span := recordSpan("span-a", map[string]string{"db.statement": longStatement, "http.route": "/charge"})
|
||||
|
||||
record := NewSpanRecord(&span)
|
||||
|
||||
assert.Equal(t, "Error", record.StatusCode)
|
||||
assert.Equal(t, "/charge", record.Attributes["http.route"])
|
||||
assert.InDelta(t, 502, record.Attributes["http.response.status_code"], 0)
|
||||
assert.Len(t, []rune(record.Attributes["db.statement"].(string)), spanRecordValueChars+1)
|
||||
assert.Equal(t, []TruncatedField{{Field: "attributes.db.statement", Length: uint64(len(longStatement))}}, record.Truncated)
|
||||
require.Len(t, record.Events, 1)
|
||||
assert.True(t, record.Events[0].IsError)
|
||||
assert.Equal(t, []Link{{TraceID: "other-trace", SpanID: "producer"}}, record.Links)
|
||||
})
|
||||
|
||||
t.Run("MultiByteValue_IsCutOnACharacterBoundary", func(t *testing.T) {
|
||||
span := recordSpan("span-c", map[string]string{"note": strings.Repeat("é", 600)})
|
||||
|
||||
record := NewSpanRecord(&span)
|
||||
|
||||
assert.Equal(t, strings.Repeat("é", spanRecordValueChars)+truncationMark, record.Attributes["note"])
|
||||
assert.Equal(t, uint64(600), record.Truncated[0].Length)
|
||||
})
|
||||
|
||||
t.Run("RecordOverBudget_ShrinksValuesFurther", func(t *testing.T) {
|
||||
attributes := make(map[string]string, 80)
|
||||
for i := range 80 {
|
||||
attributes["key."+strings.Repeat("k", i+1)] = strings.Repeat("v", 480)
|
||||
}
|
||||
span := recordSpan("span-d", attributes)
|
||||
|
||||
record := NewSpanRecord(&span)
|
||||
|
||||
assert.LessOrEqual(t, jsonChars(record), spanRecordChars)
|
||||
assert.Len(t, record.Truncated, 80)
|
||||
assert.Equal(t, uint64(480), record.Truncated[0].Length)
|
||||
})
|
||||
}
|
||||
|
||||
func TestSmallTraceRecords(t *testing.T) {
|
||||
small := []StorableSpan{recordSpan("span-a", map[string]string{"k": "v"}), recordSpan("span-b", map[string]string{"k": "v"})}
|
||||
records, ok := SmallTraceRecords(small)
|
||||
require.True(t, ok)
|
||||
assert.Len(t, records, 2)
|
||||
|
||||
large := make([]StorableSpan, 0, 60)
|
||||
for range 60 {
|
||||
large = append(large, recordSpan("span", map[string]string{"payload": strings.Repeat("x", 450)}))
|
||||
}
|
||||
_, ok = SmallTraceRecords(large)
|
||||
assert.False(t, ok)
|
||||
|
||||
assert.True(t, IsSmallTrace(inspectTreeSpans))
|
||||
assert.False(t, IsSmallTrace(inspectTreeSpans+1))
|
||||
}
|
||||
288
tests/integration/tests/tracedetail/05_inspect.py
Normal file
288
tests/integration/tests/tracedetail/05_inspect.py
Normal file
@@ -0,0 +1,288 @@
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.querier import get_attribute_backend_now
|
||||
from fixtures.traces import TraceIdGenerator, Traces, TracesEvent, TracesKind, TracesLink, TracesRefType, TracesStatusCode
|
||||
|
||||
OVERVIEW_LISTS = ("errors", "unmarkedFailures", "slowest", "expensive", "links")
|
||||
LIST_SIZE = 10
|
||||
|
||||
|
||||
def get_inspect(signoz: types.SigNoz, token: str, trace_id: str, params: dict | None = None) -> requests.Response:
|
||||
return requests.get(
|
||||
signoz.self.host_configs["8080"].get(f"/api/v1/traces/{trace_id}/inspect"),
|
||||
params=params,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
|
||||
def get_overview(signoz: types.SigNoz, token: str, trace_id: str, params: dict | None = None) -> dict:
|
||||
response = get_inspect(signoz, token, trace_id, params)
|
||||
assert response.status_code == HTTPStatus.OK, f"{params}: {response.text}"
|
||||
data = response.json()["data"]
|
||||
assert data["type"] == "overview"
|
||||
return data["overview"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("attribute_backend", ["map", "json"])
|
||||
def test_inspect_overview(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
attribute_backend: str,
|
||||
) -> None:
|
||||
"""The overview of a small trace: identity with per-service totals and gen_ai usage, where the
|
||||
error started, the failures not marked as errors, span links, the most expensive gen_ai spans and
|
||||
the whole tree with every span's record. Attributes are written to one layout only, so a read
|
||||
from the wrong column loses the tokens, the cost and the record's attributes."""
|
||||
now = get_attribute_backend_now(signoz, attribute_backend).replace(second=0, microsecond=0)
|
||||
write_mode = "json_only" if attribute_backend == "json" else "legacy_only"
|
||||
trace_id = TraceIdGenerator.trace_id()
|
||||
other_trace_id = TraceIdGenerator.trace_id()
|
||||
names = ("root", "place_order", "charge_card", "inventory_http", "inventory_grpc", "render", "consume", "model", "tool")
|
||||
ids = {name: TraceIdGenerator.span_id() for name in names}
|
||||
gateway, orders, agent = (f"td-inspect-{service}-{attribute_backend}" for service in ("gateway", "orders", "agent"))
|
||||
|
||||
def span(key: str, parent: str, name: str, service: str, offset_ms: int, duration_ms: int, **kwargs) -> Traces:
|
||||
return Traces(
|
||||
timestamp=now - timedelta(seconds=10) + timedelta(milliseconds=offset_ms),
|
||||
duration=timedelta(milliseconds=duration_ms),
|
||||
trace_id=trace_id,
|
||||
span_id=ids[key],
|
||||
parent_span_id=ids[parent] if parent else "",
|
||||
name=name,
|
||||
resources={"service.name": service},
|
||||
attribute_write_mode=write_mode,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
render = span("render", "root", "render_receipt", gateway, 700, 50, events=[TracesEvent("exception", now, {"exception.message": "template missing, fell back"})])
|
||||
# The collector marks an exception event with isError; the fixture does not write it.
|
||||
render._events_db = [json.dumps({"name": "exception", "timeUnixNano": 1, "attributeMap": {"exception.message": "template missing, fell back"}, "isError": True}, separators=(",", ":"))] # pylint: disable=protected-access
|
||||
|
||||
insert_traces(
|
||||
[
|
||||
span("root", "", "POST /checkout", gateway, 0, 1000, kind=TracesKind.SPAN_KIND_SERVER, status_code=TracesStatusCode.STATUS_CODE_ERROR, attributes={"http.response.status_code": 500}),
|
||||
# the error is carried up from charge_card through place_order to the root
|
||||
span("place_order", "root", "place_order", orders, 10, 600, status_code=TracesStatusCode.STATUS_CODE_ERROR),
|
||||
span("charge_card", "place_order", "charge_card", orders, 20, 300, kind=TracesKind.SPAN_KIND_CLIENT, status_code=TracesStatusCode.STATUS_CODE_ERROR, status_message="card declined", attributes={"payment.provider": "worldpay"}),
|
||||
# three failures that do not carry the error flag
|
||||
span("inventory_http", "place_order", "GET /inventory", orders, 330, 40, kind=TracesKind.SPAN_KIND_CLIENT, attributes={"http.response.status_code": 404}),
|
||||
span("inventory_grpc", "place_order", "inventory.Check", orders, 380, 40, kind=TracesKind.SPAN_KIND_CLIENT, attributes={"rpc.grpc.status_code": 5}),
|
||||
render,
|
||||
# a link out to another trace
|
||||
span("consume", "root", "consume order.created", orders, 760, 20, kind=TracesKind.SPAN_KIND_CONSUMER, links=[TracesLink(trace_id=other_trace_id, span_id="aaaaaaaaaaaaaaaa", ref_type=TracesRefType.REF_TYPE_FOLLOWS_FROM)]),
|
||||
span("model", "root", "chat gpt-4o-mini", agent, 800, 100, kind=TracesKind.SPAN_KIND_CLIENT, attributes={"gen_ai.request.model": "gpt-4o-mini", "gen_ai.usage.input_tokens": 100, "gen_ai.usage.output_tokens": 20, "signoz.gen_ai.usage.tokens.cost": 0.01}),
|
||||
span("tool", "root", "execute_tool", agent, 910, 30, attributes={"gen_ai.tool.name": "get_weather"}),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
overview = get_overview(signoz, token, trace_id)
|
||||
|
||||
identity = overview["identity"]
|
||||
assert identity["traceId"] == trace_id
|
||||
assert identity["spanCount"] == 9
|
||||
assert identity["errorSpanCount"] == 3
|
||||
assert identity["rootCount"] == 1
|
||||
assert identity["hasMissingSpans"] is False
|
||||
assert identity["durationNano"] == 1_000_000_000
|
||||
assert identity["root"]["spanId"] == ids["root"]
|
||||
assert identity["root"]["name"] == "POST /checkout"
|
||||
assert identity["root"]["responseStatusCode"] == "500"
|
||||
assert identity["serviceCount"] == 3
|
||||
assert {service["serviceName"]: (service["spanCount"], service["errorSpanCount"]) for service in identity["services"]} == {gateway: (2, 1), orders: (5, 2), agent: (2, 0)}
|
||||
assert identity["ai"]["modelCallCount"] == 1
|
||||
assert identity["ai"]["toolCallCount"] == 1
|
||||
assert identity["ai"]["tokens"] == {"input": 100, "output": 20, "cacheRead": 0, "cacheWrite": 0, "reasoning": 0}
|
||||
assert identity["ai"]["totalCost"] == pytest.approx(0.01)
|
||||
|
||||
# one entry for the error, at the span that threw it
|
||||
errors = overview["errors"]
|
||||
assert errors["total"] == 1
|
||||
assert "nextCursor" not in errors
|
||||
assert errors["items"][0]["span"]["spanId"] == ids["charge_card"]
|
||||
assert errors["items"][0]["parent"]["spanId"] == ids["place_order"]
|
||||
assert errors["items"][0]["statusMessage"] == "card declined"
|
||||
assert errors["items"][0]["errorAncestorCount"] == 2
|
||||
|
||||
unmarked = overview["unmarkedFailures"]
|
||||
assert unmarked["total"] == 3
|
||||
assert {item["span"]["spanId"]: item["reason"] for item in unmarked["items"]} == {ids["inventory_http"]: "http_4xx", ids["inventory_grpc"]: "grpc_status", ids["render"]: "exception_event"}
|
||||
|
||||
links = overview["links"]
|
||||
assert links["total"] == 1
|
||||
assert links["items"][0]["span"]["spanId"] == ids["consume"]
|
||||
assert links["items"][0]["parent"]["spanId"] == ids["root"]
|
||||
assert links["items"][0]["linkCount"] == 1
|
||||
assert links["items"][0]["links"] == [{"traceId": other_trace_id, "spanId": "aaaaaaaaaaaaaaaa"}]
|
||||
|
||||
expensive = overview["expensive"]
|
||||
assert expensive["total"] == 2
|
||||
assert expensive["items"][0]["span"]["spanId"] == ids["model"]
|
||||
assert expensive["items"][0]["cost"] == pytest.approx(0.01)
|
||||
assert (expensive["items"][0]["inputTokens"], expensive["items"][0]["outputTokens"]) == (100, 20)
|
||||
|
||||
# own time is the duration minus what the children cover: 300 ms of charge_card is all its own
|
||||
slowest = overview["slowest"]
|
||||
assert slowest["total"] == 9
|
||||
assert slowest["items"][0]["span"]["spanId"] == ids["charge_card"]
|
||||
assert slowest["items"][0]["ownTimeNano"] == 300_000_000
|
||||
assert slowest["items"][0]["spanCount"] == 1
|
||||
|
||||
# every list item carries its direct parent, and the root has none
|
||||
for name in OVERVIEW_LISTS:
|
||||
for item in overview[name]["items"]:
|
||||
assert item.get("parent", {}).get("spanId", "") == item["span"].get("parentSpanId", ""), f"{name}: {item}"
|
||||
|
||||
tree = overview["tree"]
|
||||
assert tree["total"] == 9
|
||||
assert tree["complete"] is True
|
||||
# tree order: a span is followed by the spans below it, siblings by start time
|
||||
assert [line["spanId"] for line in tree["spans"]] == [ids[name] for name in ("root", "place_order", "charge_card", "inventory_http", "inventory_grpc", "render", "consume", "model", "tool")]
|
||||
place_order = tree["spans"][1]
|
||||
assert (place_order["level"], place_order["childCount"], place_order["subtreeSpanCount"]) == (1, 3, 3)
|
||||
assert place_order["ownTimeNano"] == 220_000_000
|
||||
|
||||
# a small trace carries every span's record
|
||||
records = {record["spanId"]: record for record in tree["records"]}
|
||||
assert set(records) == set(ids.values())
|
||||
assert records[ids["charge_card"]]["attributes"]["payment.provider"] == "worldpay"
|
||||
assert records[ids["charge_card"]]["statusMessage"] == "card declined"
|
||||
assert records[ids["charge_card"]]["resource"]["service.name"] == orders
|
||||
assert records[ids["consume"]]["links"] == [{"traceId": other_trace_id, "spanId": "aaaaaaaaaaaaaaaa"}]
|
||||
assert records[ids["render"]]["events"][0]["name"] == "exception"
|
||||
|
||||
|
||||
def test_inspect_pages_every_list(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""Every list of the overview returns 10 items and a cursor. The cursor alone returns the
|
||||
rest of that list, and nothing else of the overview."""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
trace_id = TraceIdGenerator.trace_id()
|
||||
root_id = TraceIdGenerator.span_id()
|
||||
resources = {"service.name": "td-inspect-pages"}
|
||||
spans = [Traces(timestamp=now - timedelta(seconds=30), duration=timedelta(seconds=20), trace_id=trace_id, span_id=root_id, name="POST /batch", kind=TracesKind.SPAN_KIND_SERVER, resources=resources)]
|
||||
expected: dict[str, list[str]] = {name: [] for name in OVERVIEW_LISTS}
|
||||
|
||||
def child(group: str, i: int, offset_ms: int, **kwargs) -> None:
|
||||
span_id = TraceIdGenerator.span_id()
|
||||
expected[group].append(span_id)
|
||||
# distinct names keep each span its own entry in the slowest list; distinct durations fix its order
|
||||
spans.append(Traces(timestamp=now - timedelta(seconds=29) + timedelta(milliseconds=offset_ms), duration=timedelta(milliseconds=100 + offset_ms), trace_id=trace_id, span_id=span_id, parent_span_id=root_id, name=f"{group}-{i:02d}", resources=resources, **kwargs))
|
||||
|
||||
for i in range(12):
|
||||
child("errors", i, i, status_code=TracesStatusCode.STATUS_CODE_ERROR, status_message=f"failure {i:02d}")
|
||||
child("unmarkedFailures", i, 20 + i, attributes={"http.response.status_code": 404})
|
||||
child("expensive", i, 40 + i, attributes={"gen_ai.request.model": "gpt-4o-mini", "signoz.gen_ai.usage.tokens.cost": (i + 1) / 100})
|
||||
child("links", i, 60 + i, links=[TracesLink(trace_id=TraceIdGenerator.trace_id(), span_id=TraceIdGenerator.span_id(), ref_type=TracesRefType.REF_TYPE_FOLLOWS_FROM)])
|
||||
insert_traces(spans)
|
||||
# the slowest list ranks every span: the 48 children by duration, the longest first, then the root
|
||||
by_duration = sorted(spans[1:], key=lambda span: int(span.duration_nano), reverse=True)
|
||||
expected["slowest"] = [span.span_id for span in by_duration]
|
||||
expected["expensive"].reverse()
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
overview = get_overview(signoz, token, trace_id)
|
||||
assert overview["identity"]["spanCount"] == 49
|
||||
|
||||
for name in OVERVIEW_LISTS:
|
||||
first = overview[name]
|
||||
assert len(first["items"]) == LIST_SIZE, name
|
||||
assert first["nextCursor"], name
|
||||
seen = [item["span"]["spanId"] for item in first["items"]]
|
||||
cursor = first["nextCursor"]
|
||||
for _ in range(10):
|
||||
page = get_overview(signoz, token, trace_id, {"cursor": cursor})
|
||||
assert list(page) == [name], f"{name}: a page returned {list(page)}"
|
||||
assert page[name]["total"] == first["total"]
|
||||
seen += [item["span"]["spanId"] for item in page[name]["items"]]
|
||||
cursor = page[name].get("nextCursor")
|
||||
if not cursor:
|
||||
break
|
||||
assert len(seen) == first["total"], name
|
||||
assert len(set(seen)) == len(seen), f"{name}: a span was returned twice"
|
||||
if name == "slowest":
|
||||
# the root's own time is what its children leave of it, so only its place is not asserted
|
||||
assert [span_id for span_id in seen if span_id != root_id] == expected[name]
|
||||
assert first["total"] == 49
|
||||
else:
|
||||
assert seen == expected[name], name
|
||||
assert first["total"] == 12
|
||||
|
||||
# the second page of errors carries the status messages of its own spans
|
||||
second = get_overview(signoz, token, trace_id, {"cursor": overview["errors"]["nextCursor"]})
|
||||
assert [item["statusMessage"] for item in second["errors"]["items"]] == ["failure 10", "failure 11"]
|
||||
|
||||
|
||||
def test_inspect_large_trace_shows_the_top_of_the_tree(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""Over 50 spans the tree holds the first 50 in time order, the records are left out, and
|
||||
same-name spans under one parent are one entry of the slowest list."""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
trace_id = TraceIdGenerator.trace_id()
|
||||
root_id = TraceIdGenerator.span_id()
|
||||
resources = {"service.name": "td-inspect-large"}
|
||||
spans = [Traces(timestamp=now - timedelta(seconds=30), duration=timedelta(seconds=20), trace_id=trace_id, span_id=root_id, name="POST /import", kind=TracesKind.SPAN_KIND_SERVER, resources=resources)]
|
||||
for i in range(150):
|
||||
spans.append(Traces(timestamp=now - timedelta(seconds=29) + timedelta(milliseconds=i * 10), duration=timedelta(milliseconds=5), trace_id=trace_id, span_id=TraceIdGenerator.span_id(), parent_span_id=root_id, name="write_item", resources=resources))
|
||||
insert_traces(spans)
|
||||
in_time_order = [span.span_id for span in spans]
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
overview = get_overview(signoz, token, trace_id)
|
||||
|
||||
assert overview["identity"]["spanCount"] == 151
|
||||
tree = overview["tree"]
|
||||
assert tree["total"] == 151
|
||||
assert tree["complete"] is False
|
||||
assert "records" not in tree
|
||||
assert [line["spanId"] for line in tree["spans"]] == in_time_order[:50]
|
||||
assert tree["spans"][0]["childCount"] == 150
|
||||
|
||||
slowest = overview["slowest"]
|
||||
assert slowest["total"] == 2
|
||||
group = next(item for item in slowest["items"] if item["span"]["name"] == "write_item")
|
||||
assert group["spanCount"] == 150
|
||||
assert group["ownTimeNano"] == 150 * 5_000_000
|
||||
assert group["parent"]["spanId"] == root_id
|
||||
assert "expensive" not in overview
|
||||
|
||||
|
||||
def test_inspect_rejects_invalid_requests(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
trace_id = TraceIdGenerator.trace_id()
|
||||
|
||||
for params in (
|
||||
{"depth": 3},
|
||||
{"limit": 10},
|
||||
{"selectFields": "http.route"},
|
||||
{"cursor": "not-a-cursor"},
|
||||
{"depth": "all"},
|
||||
):
|
||||
response = get_inspect(signoz, token, trace_id, params)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST, f"{params}: {response.text}"
|
||||
|
||||
missing = get_inspect(signoz, token, trace_id)
|
||||
assert missing.status_code == HTTPStatus.NOT_FOUND, missing.text
|
||||
Reference in New Issue
Block a user